Merge tag 'drm-for-v4.13' of git://people.freedesktop.org/~airlied/linux

Pull drm updates from Dave Airlie: "This is the main pull request for the drm, I think I've got one later driver pull for mediatek SoC driver, I'm undecided on if it needs to go to you yet. Otherwise summary below: Core drm: - Atomic add driver private objects - Deprecate preclose hook in modern drivers - MST bandwidth tracking - Use kvmalloc in more places - Add mode_valid hook for crtc/encoder/bridge - Reduce sync_file construction time - Documentation updates - New DRM synchronisation object support New drivers: - pl111 - pl111 CLCD display controller Panel: - Innolux P079ZCA panel driver - Add NL12880B20-05, NL192108AC18-02D, P320HVN03 panels - panel-samsung-s6e3ha2: Add s6e3hf2 panel support i915: - SKL+ watermark fixes - G4x/G33 reset improvements - DP AUX backlight improvements - Buffer based GuC/host communication - New getparam for (sub)slice infomation - Cannonlake and Coffeelake initial patches - Execbuf optimisations radeon/amdgpu: - Lots of Vega10 bug fixes - Preliminary raven support - KIQ support for compute rings - MEC queue management rework - DCE6 Audio support - SR-IOV improvements - Better radeon/amdgpu selection support nouveau: - HDMI stereoscopic support - Display code rework for >= GM20x GPUs msm: - GEM rework for fine-grained locking - Per-process pagetable work - HDMI fixes for Snapdragon 820. vc4: - Remove 256MB CMA limit from vc4 - Add out-fence support - Add support for cygnus - Get/set tiling ioctls support - Add T-format tiling support for scanout zte: - add VGA support. etnaviv: - Thermal throttle support for newer GPUs - Restore userspace buffer cache performance - dma-buf sync fix stm: - add stm32f429 display support exynos: - Rework vblank handling - Fixup sw-trigger code sun4i: - V3s display engine support - HDMI support for older SoCs - Preliminary work on dual-pipeline SoCs. rcar-du: - VSP work imx-drm: - Remove counter load enable from PRE - Double read/write reduction flag support tegra: - Documentation for the host1x and drm driver. - Lots of staging ioctl fixes due to grate project work. omapdrm: - dma-buf fence support - TILER rotation fixes" * tag 'drm-for-v4.13' of git://people.freedesktop.org/~airlied/linux: (1270 commits) drm: Remove unused drm_file parameter to drm_syncobj_replace_fence() drm/amd/powerplay: fix bug fail to remove sysfs when rmmod amdgpu. amdgpu: Set cik/si_support to 1 by default if radeon isn't built drm/amdgpu/gfx9: fix driver reload with KIQ drm/amdgpu/gfx8: fix driver reload with KIQ drm/amdgpu: Don't call amd_powerplay_destroy() if we don't have powerplay drm/ttm: Fix use-after-free in ttm_bo_clean_mm drm/amd/amdgpu: move get memory type function from early init to sw init drm/amdgpu/cgs: always set reference clock in mode_info drm/amdgpu: fix vblank_time when displays are off drm/amd/powerplay: power value format change for Vega10 drm/amdgpu/gfx9: support the amdgpu.disable_cu option drm/amd/powerplay: change PPSMC_MSG_GetCurrPkgPwr for Vega10 drm/amdgpu: Make amdgpu_cs_parser_init static (v2) drm/amdgpu/cs: fix a typo in a comment drm/amdgpu: Fix the exported always on CU bitmap drm/amdgpu/gfx9: gfx_v9_0_enable_gfx_static_mg_power_gating() can be static drm/amdgpu/psp: upper_32_bits/lower_32_bits for address setup drm/amd/powerplay/cz: print message if smc message fails drm/amdgpu: fix typo in amdgpu_debugfs_test_ib_init ...
author: Linus Torvalds <torvalds@linux-foundation.org> 2017-07-09 18:48:37 -0700
committer: Linus Torvalds <torvalds@linux-foundation.org> 2017-07-09 18:48:37 -0700
commit: af3c8d98508d37541d4bf57f13a984a7f73a328c (patch)
tree: e8dd974d6ebccd38b1e373be8a5e4a2f8bf3c6ce /drivers/gpu/drm/i915
parent: afs: Add metadata xattrs (diff)
parent: drm: Remove unused drm_file parameter to drm_syncobj_replace_fence() (diff)
download: linux-dev-af3c8d98508d37541d4bf57f13a984a7f73a328c.tar.xz
linux-dev-af3c8d98508d37541d4bf57f13a984a7f73a328c.zip
161 files changed, 44219 insertions, 6167 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index b00edd3b8800..78c5c049a347 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -61,6 +61,18 @@ config DRM_I915_SW_FENCE_DEBUG_OBJECTS
 
           If in doubt, say "N".
 
+config DRM_I915_SW_FENCE_CHECK_DAG
+        bool "Enable additional driver debugging for detecting dependency cycles"
+        depends on DRM_I915
+        default n
+        help
+          Choose this option to turn on extra driver debugging that may affect
+          performance but will catch some internal issues.
+
+          Recommended for driver developers only.
+
+          If in doubt, say "N".
+
 config DRM_I915_SELFTEST
 	bool "Enable selftests upon driver load"
 	depends on DRM_I915
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 2cf04504e494..f8227318dcaf 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -16,6 +16,7 @@ i915-y := i915_drv.o \
 	  i915_params.o \
 	  i915_pci.o \
           i915_suspend.o \
+	  i915_syncmap.o \
 	  i915_sw_fence.o \
 	  i915_sysfs.o \
 	  intel_csr.o \
@@ -57,6 +58,7 @@ i915-y += i915_cmd_parser.o \
 
 # general-purpose microcontroller (GuC) support
 i915-y += intel_uc.o \
+	  intel_guc_ct.o \
 	  intel_guc_log.o \
 	  intel_guc_loader.o \
 	  intel_huc.o \
@@ -127,7 +129,16 @@ i915-y += i915_vgpu.o
 
 # perf code
 i915-y += i915_perf.o \
-	  i915_oa_hsw.o
+	  i915_oa_hsw.o \
+	  i915_oa_bdw.o \
+	  i915_oa_chv.o \
+	  i915_oa_sklgt2.o \
+	  i915_oa_sklgt3.o \
+	  i915_oa_sklgt4.o \
+	  i915_oa_bxt.o \
+	  i915_oa_kblgt2.o \
+	  i915_oa_kblgt3.o \
+	  i915_oa_glk.o
 
 ifeq ($(CONFIG_DRM_I915_GVT),y)
 i915-y += intel_gvt.o
diff --git a/drivers/gpu/drm/i915/dvo_ch7017.c b/drivers/gpu/drm/i915/dvo_ch7017.c
index b3c7c199200c..80b3e16cf48c 100644
--- a/drivers/gpu/drm/i915/dvo_ch7017.c
+++ b/drivers/gpu/drm/i915/dvo_ch7017.c
@@ -280,10 +280,10 @@ static void ch7017_mode_set(struct intel_dvo_device *dvo,
 			(0 << CH7017_PHASE_DETECTOR_SHIFT);
 	} else {
 		outputs_enable = CH7017_LVDS_CHANNEL_A | CH7017_CHARGE_PUMP_HIGH;
-		lvds_pll_feedback_div = CH7017_LVDS_PLL_FEEDBACK_DEFAULT_RESERVED |
+		lvds_pll_feedback_div =
+			CH7017_LVDS_PLL_FEEDBACK_DEFAULT_RESERVED |
 			(2 << CH7017_LVDS_PLL_FEED_BACK_DIVIDER_SHIFT) |
 			(3 << CH7017_LVDS_PLL_FEED_FORWARD_DIVIDER_SHIFT);
-		lvds_pll_feedback_div = 35;
 		lvds_control_2 = (3 << CH7017_LOOP_FILTER_SHIFT) |
 			(0 << CH7017_PHASE_DETECTOR_SHIFT);
 		if (1) { /* XXX: dual channel panel detection.  Assume yes for now. */
diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c
index 44b3159f2fe8..7aeeffd2428b 100644
--- a/drivers/gpu/drm/i915/dvo_ch7xxx.c
+++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c
@@ -217,9 +217,8 @@ static bool ch7xxx_init(struct intel_dvo_device *dvo,
 
 	name = ch7xxx_get_id(vendor);
 	if (!name) {
-		DRM_DEBUG_KMS("ch7xxx not detected; got 0x%02x from %s "
-				"slave %d.\n",
-			  vendor, adapter->name, dvo->slave_addr);
+		DRM_DEBUG_KMS("ch7xxx not detected; got VID 0x%02x from %s slave %d.\n",
+			      vendor, adapter->name, dvo->slave_addr);
 		goto out;
 	}
 
@@ -229,9 +228,8 @@ static bool ch7xxx_init(struct intel_dvo_device *dvo,
 
 	devid = ch7xxx_get_did(device);
 	if (!devid) {
-		DRM_DEBUG_KMS("ch7xxx not detected; got 0x%02x from %s "
-				"slave %d.\n",
-			  vendor, adapter->name, dvo->slave_addr);
+		DRM_DEBUG_KMS("ch7xxx not detected; got DID 0x%02x from %s slave %d.\n",
+			      device, adapter->name, dvo->slave_addr);
 		goto out;
 	}
 
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index b123c20e2097..f5486cb94818 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -3,6 +3,6 @@ GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
 	interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
 	execlist.o scheduler.o sched_policy.o render.o cmd_parser.o
 
-ccflags-y				+= -I$(src) -I$(src)/$(GVT_DIR) -Wall
+ccflags-y				+= -I$(src) -I$(src)/$(GVT_DIR)
 i915-y					+= $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
 obj-$(CONFIG_DRM_I915_GVT_KVMGT)	+= $(GVT_DIR)/kvmgt.o
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 41b2c3aaa04a..51241de5e7a7 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -2414,53 +2414,13 @@ static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e)
 	hash_add(gvt->cmd_table, &e->hlist, e->info->opcode);
 }
 
-#define GVT_MAX_CMD_LENGTH     20  /* In Dword */
-
-static void trace_cs_command(struct parser_exec_state *s,
-		cycles_t cost_pre_cmd_handler, cycles_t cost_cmd_handler)
-{
-	/* This buffer is used by ftrace to store all commands copied from
-	 * guest gma space. Sometimes commands can cross pages, this should
-	 * not be handled in ftrace logic. So this is just used as a
-	 * 'bounce buffer'
-	 */
-	u32 cmd_trace_buf[GVT_MAX_CMD_LENGTH];
-	int i;
-	u32 cmd_len = cmd_length(s);
-	/* The chosen value of GVT_MAX_CMD_LENGTH are just based on
-	 * following two considerations:
-	 * 1) From observation, most common ring commands is not that long.
-	 *    But there are execeptions. So it indeed makes sence to observe
-	 *    longer commands.
-	 * 2) From the performance and debugging point of view, dumping all
-	 *    contents of very commands is not necessary.
-	 * We mgith shrink GVT_MAX_CMD_LENGTH or remove this trace event in
-	 * future for performance considerations.
-	 */
-	if (unlikely(cmd_len > GVT_MAX_CMD_LENGTH)) {
-		gvt_dbg_cmd("cmd length exceed tracing limitation!\n");
-		cmd_len = GVT_MAX_CMD_LENGTH;
-	}
-
-	for (i = 0; i < cmd_len; i++)
-		cmd_trace_buf[i] = cmd_val(s, i);
-
-	trace_gvt_command(s->vgpu->id, s->ring_id, s->ip_gma, cmd_trace_buf,
-			cmd_len, s->buf_type == RING_BUFFER_INSTRUCTION,
-			cost_pre_cmd_handler, cost_cmd_handler);
-}
-
 /* call the cmd handler, and advance ip */
 static int cmd_parser_exec(struct parser_exec_state *s)
 {
+	struct intel_vgpu *vgpu = s->vgpu;
 	struct cmd_info *info;
 	u32 cmd;
 	int ret = 0;
-	cycles_t t0, t1, t2;
-	struct parser_exec_state s_before_advance_custom;
-	struct intel_vgpu *vgpu = s->vgpu;
-
-	t0 = get_cycles();
 
 	cmd = cmd_val(s, 0);
 
@@ -2471,13 +2431,10 @@ static int cmd_parser_exec(struct parser_exec_state *s)
 		return -EINVAL;
 	}
 
-	gvt_dbg_cmd("%s\n", info->name);
-
 	s->info = info;
 
-	t1 = get_cycles();
-
-	s_before_advance_custom = *s;
+	trace_gvt_command(vgpu->id, s->ring_id, s->ip_gma, s->ip_va,
+			  cmd_length(s), s->buf_type);
 
 	if (info->handler) {
 		ret = info->handler(s);
@@ -2486,9 +2443,6 @@ static int cmd_parser_exec(struct parser_exec_state *s)
 			return ret;
 		}
 	}
-	t2 = get_cycles();
-
-	trace_cs_command(&s_before_advance_custom, t1 - t0, t2 - t1);
 
 	if (!(info->flag & F_IP_ADVANCE_CUSTOM)) {
 		ret = cmd_advance_default(s);
@@ -2522,8 +2476,6 @@ static int command_scan(struct parser_exec_state *s,
 	gma_tail = rb_start + rb_tail;
 	gma_bottom = rb_start +  rb_len;
 
-	gvt_dbg_cmd("scan_start: start=%lx end=%lx\n", gma_head, gma_tail);
-
 	while (s->ip_gma != gma_tail) {
 		if (s->buf_type == RING_BUFFER_INSTRUCTION) {
 			if (!(s->ip_gma >= rb_start) ||
@@ -2552,8 +2504,6 @@ static int command_scan(struct parser_exec_state *s,
 		}
 	}
 
-	gvt_dbg_cmd("scan_end\n");
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
index 24fe04d6307b..700050556242 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.c
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -708,53 +708,43 @@ static int submit_context(struct intel_vgpu *vgpu, int ring_id,
 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 {
 	struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
-	struct execlist_ctx_descriptor_format *desc[2], valid_desc[2];
-	unsigned long valid_desc_bitmap = 0;
-	bool emulate_schedule_in = true;
-	int ret;
-	int i;
+	struct execlist_ctx_descriptor_format desc[2];
+	int i, ret;
 
-	memset(valid_desc, 0, sizeof(valid_desc));
+	desc[0] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
+	desc[1] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
 
-	desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
-	desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+	if (!desc[0].valid) {
+		gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n");
+		goto inv_desc;
+	}
 
-	for (i = 0; i < 2; i++) {
-		if (!desc[i]->valid)
+	for (i = 0; i < ARRAY_SIZE(desc); i++) {
+		if (!desc[i].valid)
 			continue;
-
-		if (!desc[i]->privilege_access) {
+		if (!desc[i].privilege_access) {
 			gvt_vgpu_err("unexpected GGTT elsp submission\n");
-			return -EINVAL;
+			goto inv_desc;
 		}
-
-		/* TODO: add another guest context checks here. */
-		set_bit(i, &valid_desc_bitmap);
-		valid_desc[i] = *desc[i];
-	}
-
-	if (!valid_desc_bitmap) {
-		gvt_vgpu_err("no valid desc in a elsp submission\n");
-		return -EINVAL;
-	}
-
-	if (!test_bit(0, (void *)&valid_desc_bitmap) &&
-			test_bit(1, (void *)&valid_desc_bitmap)) {
-		gvt_vgpu_err("weird elsp submission, desc 0 is not valid\n");
-		return -EINVAL;
 	}
 
 	/* submit workload */
-	for_each_set_bit(i, (void *)&valid_desc_bitmap, 2) {
-		ret = submit_context(vgpu, ring_id, &valid_desc[i],
-				emulate_schedule_in);
+	for (i = 0; i < ARRAY_SIZE(desc); i++) {
+		if (!desc[i].valid)
+			continue;
+		ret = submit_context(vgpu, ring_id, &desc[i], i == 0);
 		if (ret) {
-			gvt_vgpu_err("fail to schedule workload\n");
+			gvt_vgpu_err("failed to submit desc %d\n", i);
 			return ret;
 		}
-		emulate_schedule_in = false;
 	}
+
 	return 0;
+
+inv_desc:
+	gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n",
+		     desc[0].udw, desc[0].ldw, desc[1].udw, desc[1].ldw);
+	return -EINVAL;
 }
 
 static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c
index dce8d15f706f..5dad9298b2d5 100644
--- a/drivers/gpu/drm/i915/gvt/firmware.c
+++ b/drivers/gpu/drm/i915/gvt/firmware.c
@@ -102,13 +102,8 @@ static int expose_firmware_sysfs(struct intel_gvt *gvt)
 
 	p = firmware + h->mmio_offset;
 
-	hash_for_each(gvt->mmio.mmio_info_table, i, e, node) {
-		int j;
-
-		for (j = 0; j < e->length; j += 4)
-			*(u32 *)(p + e->offset + j) =
-				I915_READ_NOTRACE(_MMIO(e->offset + j));
-	}
+	hash_for_each(gvt->mmio.mmio_info_table, i, e, node)
+		*(u32 *)(p + e->offset) = I915_READ_NOTRACE(_MMIO(e->offset));
 
 	memcpy(gvt->firmware.mmio, p, info->mmio_size);
 
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index c6f0077f590d..66374dba3b1a 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -244,15 +244,19 @@ static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index)
 	return readq(addr);
 }
 
+static void gtt_invalidate(struct drm_i915_private *dev_priv)
+{
+	mmio_hw_access_pre(dev_priv);
+	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+	mmio_hw_access_post(dev_priv);
+}
+
 static void write_pte64(struct drm_i915_private *dev_priv,
 		unsigned long index, u64 pte)
 {
 	void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
 
 	writeq(pte, addr);
-
-	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
-	POSTING_READ(GFX_FLSH_CNTL_GEN6);
 }
 
 static inline struct intel_gvt_gtt_entry *gtt_get_entry64(void *pt,
@@ -1849,6 +1853,7 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 	}
 
 	ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index);
+	gtt_invalidate(gvt->dev_priv);
 	ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
 	return 0;
 }
@@ -2301,8 +2306,6 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
 	u32 num_entries;
 	struct intel_gvt_gtt_entry e;
 
-	intel_runtime_pm_get(dev_priv);
-
 	memset(&e, 0, sizeof(struct intel_gvt_gtt_entry));
 	e.type = GTT_TYPE_GGTT_PTE;
 	ops->set_pfn(&e, gvt->gtt.scratch_ggtt_mfn);
@@ -2318,7 +2321,7 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
 	for (offset = 0; offset < num_entries; offset++)
 		ops->set_entry(NULL, &e, index + offset, false, 0, vgpu);
 
-	intel_runtime_pm_put(dev_priv);
+	gtt_invalidate(dev_priv);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index 7dea5e5d5567..c27c6838eaca 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -147,7 +147,9 @@ static int gvt_service_thread(void *data)
 			mutex_unlock(&gvt->lock);
 		}
 
-		if (test_and_clear_bit(INTEL_GVT_REQUEST_SCHED,
+		if (test_bit(INTEL_GVT_REQUEST_SCHED,
+				(void *)&gvt->service_request) ||
+			test_bit(INTEL_GVT_REQUEST_EVENT_SCHED,
 					(void *)&gvt->service_request)) {
 			intel_gvt_schedule(gvt);
 		}
@@ -244,7 +246,7 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	gvt_dbg_core("init gvt device\n");
 
 	idr_init(&gvt->vgpu_idr);
-
+	spin_lock_init(&gvt->scheduler.mmio_context_lock);
 	mutex_init(&gvt->lock);
 	gvt->dev_priv = dev_priv;
 
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 930732e5c780..3a74e79eac2f 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -165,7 +165,6 @@ struct intel_vgpu {
 	struct list_head workload_q_head[I915_NUM_ENGINES];
 	struct kmem_cache *workloads;
 	atomic_t running_workload_num;
-	ktime_t last_ctx_submit_time;
 	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
 	struct i915_gem_context *shadow_ctx;
 
@@ -196,11 +195,27 @@ struct intel_gvt_fence {
 	unsigned long vgpu_allocated_fence_num;
 };
 
-#define INTEL_GVT_MMIO_HASH_BITS 9
+#define INTEL_GVT_MMIO_HASH_BITS 11
 
 struct intel_gvt_mmio {
-	u32 *mmio_attribute;
+	u8 *mmio_attribute;
+/* Register contains RO bits */
+#define F_RO		(1 << 0)
+/* Register contains graphics address */
+#define F_GMADR		(1 << 1)
+/* Mode mask registers with high 16 bits as the mask bits */
+#define F_MODE_MASK	(1 << 2)
+/* This reg can be accessed by GPU commands */
+#define F_CMD_ACCESS	(1 << 3)
+/* This reg has been accessed by a VM */
+#define F_ACCESSED	(1 << 4)
+/* This reg has been accessed through GPU commands */
+#define F_CMD_ACCESSED	(1 << 5)
+/* This reg could be accessed by unaligned address */
+#define F_UNALIGN	(1 << 6)
+
 	DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
+	unsigned int num_tracked_mmio;
 };
 
 struct intel_gvt_firmware {
@@ -257,7 +272,12 @@ static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
 
 enum {
 	INTEL_GVT_REQUEST_EMULATE_VBLANK = 0,
+
+	/* Scheduling trigger by timer */
 	INTEL_GVT_REQUEST_SCHED = 1,
+
+	/* Scheduling trigger by event */
+	INTEL_GVT_REQUEST_EVENT_SCHED = 2,
 };
 
 static inline void intel_gvt_request_service(struct intel_gvt *gvt,
@@ -473,6 +493,80 @@ enum {
 	GVT_FAILSAFE_INSUFFICIENT_RESOURCE,
 };
 
+static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv)
+{
+	intel_runtime_pm_get(dev_priv);
+}
+
+static inline void mmio_hw_access_post(struct drm_i915_private *dev_priv)
+{
+	intel_runtime_pm_put(dev_priv);
+}
+
+/**
+ * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+static inline void intel_gvt_mmio_set_accessed(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	gvt->mmio.mmio_attribute[offset >> 2] |= F_ACCESSED;
+}
+
+/**
+ * intel_gvt_mmio_is_cmd_accessed - mark a MMIO could be accessed by command
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+static inline bool intel_gvt_mmio_is_cmd_access(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] & F_CMD_ACCESS;
+}
+
+/**
+ * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+static inline bool intel_gvt_mmio_is_unalign(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] & F_UNALIGN;
+}
+
+/**
+ * intel_gvt_mmio_set_cmd_accessed - mark a MMIO has been accessed by command
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+static inline void intel_gvt_mmio_set_cmd_accessed(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	gvt->mmio.mmio_attribute[offset >> 2] |= F_CMD_ACCESSED;
+}
+
+/**
+ * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ * Returns:
+ * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't.
+ *
+ */
+static inline bool intel_gvt_mmio_has_mode_mask(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK;
+}
+
+#include "trace.h"
 #include "mpt.h"
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 0ffd69654592..1414d7e6148d 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -47,21 +47,6 @@
 #define PCH_PP_OFF_DELAYS _MMIO(0xc720c)
 #define PCH_PP_DIVISOR _MMIO(0xc7210)
 
-/* Register contains RO bits */
-#define F_RO		(1 << 0)
-/* Register contains graphics address */
-#define F_GMADR		(1 << 1)
-/* Mode mask registers with high 16 bits as the mask bits */
-#define F_MODE_MASK	(1 << 2)
-/* This reg can be accessed by GPU commands */
-#define F_CMD_ACCESS	(1 << 3)
-/* This reg has been accessed by a VM */
-#define F_ACCESSED	(1 << 4)
-/* This reg has been accessed through GPU commands */
-#define F_CMD_ACCESSED	(1 << 5)
-/* This reg could be accessed by unaligned address */
-#define F_UNALIGN	(1 << 6)
-
 unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt)
 {
 	if (IS_BROADWELL(gvt->dev_priv))
@@ -92,11 +77,22 @@ static void write_vreg(struct intel_vgpu *vgpu, unsigned int offset,
 	memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
 }
 
+static struct intel_gvt_mmio_info *find_mmio_info(struct intel_gvt *gvt,
+						  unsigned int offset)
+{
+	struct intel_gvt_mmio_info *e;
+
+	hash_for_each_possible(gvt->mmio.mmio_info_table, e, node, offset) {
+		if (e->offset == offset)
+			return e;
+	}
+	return NULL;
+}
+
 static int new_mmio_info(struct intel_gvt *gvt,
-		u32 offset, u32 flags, u32 size,
+		u32 offset, u8 flags, u32 size,
 		u32 addr_mask, u32 ro_mask, u32 device,
-		int (*read)(struct intel_vgpu *, unsigned int, void *, unsigned int),
-		int (*write)(struct intel_vgpu *, unsigned int, void *, unsigned int))
+		gvt_mmio_func read, gvt_mmio_func write)
 {
 	struct intel_gvt_mmio_info *info, *p;
 	u32 start, end, i;
@@ -116,13 +112,11 @@ static int new_mmio_info(struct intel_gvt *gvt,
 			return -ENOMEM;
 
 		info->offset = i;
-		p = intel_gvt_find_mmio_info(gvt, info->offset);
+		p = find_mmio_info(gvt, info->offset);
 		if (p)
 			gvt_err("dup mmio definition offset %x\n",
 				info->offset);
-		info->size = size;
-		info->length = (i + 4) < end ? 4 : (end - i);
-		info->addr_mask = addr_mask;
+
 		info->ro_mask = ro_mask;
 		info->device = device;
 		info->read = read ? read : intel_vgpu_default_mmio_read;
@@ -130,6 +124,7 @@ static int new_mmio_info(struct intel_gvt *gvt,
 		gvt->mmio.mmio_attribute[info->offset / 4] = flags;
 		INIT_HLIST_NODE(&info->node);
 		hash_add(gvt->mmio.mmio_info_table, &info->node, info->offset);
+		gvt->mmio.num_tracked_mmio++;
 	}
 	return 0;
 }
@@ -209,6 +204,7 @@ static int fence_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
 static int fence_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 		void *p_data, unsigned int bytes)
 {
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	unsigned int fence_num = offset_to_fence_num(off);
 	int ret;
 
@@ -217,8 +213,10 @@ static int fence_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 		return ret;
 	write_vreg(vgpu, off, p_data, bytes);
 
+	mmio_hw_access_pre(dev_priv);
 	intel_vgpu_write_fence(vgpu, fence_num,
 			vgpu_vreg64(vgpu, fence_num_to_offset(fence_num)));
+	mmio_hw_access_post(dev_priv);
 	return 0;
 }
 
@@ -300,6 +298,9 @@ static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 
 	intel_gvt_reset_vgpu_locked(vgpu, false, engine_mask);
 
+	/* sw will wait for the device to ack the reset request */
+	 vgpu_vreg(vgpu, offset) = 0;
+
 	return 0;
 }
 
@@ -1265,7 +1266,10 @@ static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset,
 	}
 	write_vreg(vgpu, offset, p_data, bytes);
 	/* TRTTE is not per-context */
+
+	mmio_hw_access_pre(dev_priv);
 	I915_WRITE(_MMIO(offset), vgpu_vreg(vgpu, offset));
+	mmio_hw_access_post(dev_priv);
 
 	return 0;
 }
@@ -1278,7 +1282,9 @@ static int gen9_trtt_chicken_write(struct intel_vgpu *vgpu, unsigned int offset,
 
 	if (val & 1) {
 		/* unblock hw logic */
+		mmio_hw_access_pre(dev_priv);
 		I915_WRITE(_MMIO(offset), val);
+		mmio_hw_access_post(dev_priv);
 	}
 	write_vreg(vgpu, offset, p_data, bytes);
 	return 0;
@@ -1415,7 +1421,20 @@ static int ring_timestamp_mmio_read(struct intel_vgpu *vgpu,
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 
+	mmio_hw_access_pre(dev_priv);
 	vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset));
+	mmio_hw_access_post(dev_priv);
+	return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+}
+
+static int instdone_mmio_read(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	mmio_hw_access_pre(dev_priv);
+	vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset));
+	mmio_hw_access_post(dev_priv);
 	return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
 }
 
@@ -1434,7 +1453,6 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 
 	execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data;
 	if (execlist->elsp_dwords.index == 3) {
-		vgpu->last_ctx_submit_time = ktime_get();
 		ret = intel_vgpu_submit_execlist(vgpu, ring_id);
 		if(ret)
 			gvt_vgpu_err("fail submit workload on ring %d\n",
@@ -1603,6 +1621,12 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL);
 #undef RING_REG
 
+#define RING_REG(base) (base + 0x6c)
+	MMIO_RING_DFH(RING_REG, D_ALL, 0, instdone_mmio_read, NULL);
+	MMIO_DH(RING_REG(GEN8_BSD2_RING_BASE), D_ALL, instdone_mmio_read, NULL);
+#undef RING_REG
+	MMIO_DH(GEN7_SC_INSTDONE, D_BDW_PLUS, instdone_mmio_read, NULL);
+
 	MMIO_GM_RDR(0x2148, D_ALL, NULL, NULL);
 	MMIO_GM_RDR(CCID, D_ALL, NULL, NULL);
 	MMIO_GM_RDR(0x12198, D_ALL, NULL, NULL);
@@ -1779,10 +1803,6 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(SPRSCALE(PIPE_C), D_ALL);
 	MMIO_D(SPRSURFLIVE(PIPE_C), D_ALL);
 
-	MMIO_F(LGC_PALETTE(PIPE_A, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(LGC_PALETTE(PIPE_B, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(LGC_PALETTE(PIPE_C, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
-
 	MMIO_D(HTOTAL(TRANSCODER_A), D_ALL);
 	MMIO_D(HBLANK(TRANSCODER_A), D_ALL);
 	MMIO_D(HSYNC(TRANSCODER_A), D_ALL);
@@ -2187,7 +2207,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DFH(GTFIFODBG, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(GTFIFOCTL, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DH(FORCEWAKE_MT, D_PRE_SKL, NULL, mul_force_wake_write);
-	MMIO_DH(FORCEWAKE_ACK_HSW, D_HSW | D_BDW, NULL, NULL);
+	MMIO_DH(FORCEWAKE_ACK_HSW, D_BDW, NULL, NULL);
 	MMIO_D(ECOBUS, D_ALL);
 	MMIO_DH(GEN6_RC_CONTROL, D_ALL, NULL, NULL);
 	MMIO_DH(GEN6_RC_STATE, D_ALL, NULL, NULL);
@@ -2219,22 +2239,19 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(GEN6_RC6p_THRESHOLD, D_ALL);
 	MMIO_D(GEN6_RC6pp_THRESHOLD, D_ALL);
 	MMIO_D(GEN6_PMINTRMSK, D_ALL);
-	MMIO_DH(HSW_PWR_WELL_BIOS, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_DRIVER, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_KVMR, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_DEBUG, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_CTL5, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_CTL6, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_BIOS, D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_DRIVER, D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_KVMR, D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_DEBUG, D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL5, D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL6, D_BDW, NULL, power_well_ctl_mmio_write);
 
 	MMIO_D(RSTDBYCTL, D_ALL);
 
 	MMIO_DH(GEN6_GDRST, D_ALL, NULL, gdrst_mmio_write);
 	MMIO_F(FENCE_REG_GEN6_LO(0), 0x80, 0, 0, 0, D_ALL, fence_mmio_read, fence_mmio_write);
-	MMIO_F(VGT_PVINFO_PAGE, VGT_PVINFO_SIZE, F_UNALIGN, 0, 0, D_ALL, pvinfo_mmio_read, pvinfo_mmio_write);
 	MMIO_DH(CPU_VGACNTRL, D_ALL, NULL, vga_control_mmio_write);
 
-	MMIO_F(MCHBAR_MIRROR_BASE_SNB, 0x40000, 0, 0, 0, D_ALL, NULL, NULL);
-
 	MMIO_D(TILECTL, D_ALL);
 
 	MMIO_D(GEN6_UCGCTL1, D_ALL);
@@ -2242,7 +2259,6 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 
 	MMIO_F(0x4f000, 0x90, 0, 0, 0, D_ALL, NULL, NULL);
 
-	MMIO_D(GEN6_PCODE_MAILBOX, D_PRE_BDW);
 	MMIO_D(GEN6_PCODE_DATA, D_ALL);
 	MMIO_D(0x13812c, D_ALL);
 	MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL);
@@ -2321,14 +2337,13 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(0x1a054, D_ALL);
 
 	MMIO_D(0x44070, D_ALL);
-	MMIO_DFH(0x215c, D_HSW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x215c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(0x2178, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(0x217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(0x12178, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(0x1217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
 
-	MMIO_F(0x2290, 8, F_CMD_ACCESS, 0, 0, D_HSW_PLUS, NULL, NULL);
-	MMIO_DFH(GEN7_OACONTROL, D_HSW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_F(0x2290, 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL);
 	MMIO_D(0x2b00, D_BDW_PLUS);
 	MMIO_D(0x2360, D_BDW_PLUS);
 	MMIO_F(0x5200, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
@@ -2766,7 +2781,6 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(0x72380, D_SKL_PLUS);
 	MMIO_D(0x7039c, D_SKL_PLUS);
 
-	MMIO_F(0x80000, 0x3000, 0, 0, 0, D_SKL_PLUS, NULL, NULL);
 	MMIO_D(0x8f074, D_SKL | D_KBL);
 	MMIO_D(0x8f004, D_SKL | D_KBL);
 	MMIO_D(0x8f034, D_SKL | D_KBL);
@@ -2840,26 +2854,36 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	return 0;
 }
 
-/**
- * intel_gvt_find_mmio_info - find MMIO information entry by aligned offset
- * @gvt: GVT device
- * @offset: register offset
- *
- * This function is used to find the MMIO information entry from hash table
- *
- * Returns:
- * pointer to MMIO information entry, NULL if not exists
- */
-struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
-	unsigned int offset)
-{
-	struct intel_gvt_mmio_info *e;
+/* Special MMIO blocks. */
+static struct gvt_mmio_block {
+	unsigned int device;
+	i915_reg_t   offset;
+	unsigned int size;
+	gvt_mmio_func read;
+	gvt_mmio_func write;
+} gvt_mmio_blocks[] = {
+	{D_SKL_PLUS, _MMIO(CSR_MMIO_START_RANGE), 0x3000, NULL, NULL},
+	{D_ALL, _MMIO(MCHBAR_MIRROR_BASE_SNB), 0x40000, NULL, NULL},
+	{D_ALL, _MMIO(VGT_PVINFO_PAGE), VGT_PVINFO_SIZE,
+		pvinfo_mmio_read, pvinfo_mmio_write},
+	{D_ALL, LGC_PALETTE(PIPE_A, 0), 1024, NULL, NULL},
+	{D_ALL, LGC_PALETTE(PIPE_B, 0), 1024, NULL, NULL},
+	{D_ALL, LGC_PALETTE(PIPE_C, 0), 1024, NULL, NULL},
+};
 
-	WARN_ON(!IS_ALIGNED(offset, 4));
+static struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt,
+					      unsigned int offset)
+{
+	unsigned long device = intel_gvt_get_device_type(gvt);
+	struct gvt_mmio_block *block = gvt_mmio_blocks;
+	int i;
 
-	hash_for_each_possible(gvt->mmio.mmio_info_table, e, node, offset) {
-		if (e->offset == offset)
-			return e;
+	for (i = 0; i < ARRAY_SIZE(gvt_mmio_blocks); i++, block++) {
+		if (!(device & block->device))
+			continue;
+		if (offset >= INTEL_GVT_MMIO_OFFSET(block->offset) &&
+		    offset < INTEL_GVT_MMIO_OFFSET(block->offset) + block->size)
+			return block;
 	}
 	return NULL;
 }
@@ -2899,9 +2923,10 @@ int intel_gvt_setup_mmio_info(struct intel_gvt *gvt)
 {
 	struct intel_gvt_device_info *info = &gvt->device_info;
 	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	int size = info->mmio_size / 4 * sizeof(*gvt->mmio.mmio_attribute);
 	int ret;
 
-	gvt->mmio.mmio_attribute = vzalloc(info->mmio_size);
+	gvt->mmio.mmio_attribute = vzalloc(size);
 	if (!gvt->mmio.mmio_attribute)
 		return -ENOMEM;
 
@@ -2922,77 +2947,15 @@ int intel_gvt_setup_mmio_info(struct intel_gvt *gvt)
 		if (ret)
 			goto err;
 	}
+
+	gvt_dbg_mmio("traced %u virtual mmio registers\n",
+		     gvt->mmio.num_tracked_mmio);
 	return 0;
 err:
 	intel_gvt_clean_mmio_info(gvt);
 	return ret;
 }
 
-/**
- * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed
- * @gvt: a GVT device
- * @offset: register offset
- *
- */
-void intel_gvt_mmio_set_accessed(struct intel_gvt *gvt, unsigned int offset)
-{
-	gvt->mmio.mmio_attribute[offset >> 2] |=
-		F_ACCESSED;
-}
-
-/**
- * intel_gvt_mmio_is_cmd_accessed - mark a MMIO could be accessed by command
- * @gvt: a GVT device
- * @offset: register offset
- *
- */
-bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
-		unsigned int offset)
-{
-	return gvt->mmio.mmio_attribute[offset >> 2] &
-		F_CMD_ACCESS;
-}
-
-/**
- * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned
- * @gvt: a GVT device
- * @offset: register offset
- *
- */
-bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt,
-		unsigned int offset)
-{
-	return gvt->mmio.mmio_attribute[offset >> 2] &
-		F_UNALIGN;
-}
-
-/**
- * intel_gvt_mmio_set_cmd_accessed - mark a MMIO has been accessed by command
- * @gvt: a GVT device
- * @offset: register offset
- *
- */
-void intel_gvt_mmio_set_cmd_accessed(struct intel_gvt *gvt,
-		unsigned int offset)
-{
-	gvt->mmio.mmio_attribute[offset >> 2] |=
-		F_CMD_ACCESSED;
-}
-
-/**
- * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask
- * @gvt: a GVT device
- * @offset: register offset
- *
- * Returns:
- * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't.
- *
- */
-bool intel_gvt_mmio_has_mode_mask(struct intel_gvt *gvt, unsigned int offset)
-{
-	return gvt->mmio.mmio_attribute[offset >> 2] &
-		F_MODE_MASK;
-}
 
 /**
  * intel_vgpu_default_mmio_read - default MMIO read handler
@@ -3044,3 +3007,91 @@ bool intel_gvt_in_force_nonpriv_whitelist(struct intel_gvt *gvt,
 {
 	return in_whitelist(offset);
 }
+
+/**
+ * intel_vgpu_mmio_reg_rw - emulate tracked mmio registers
+ * @vgpu: a vGPU
+ * @offset: register offset
+ * @pdata: data buffer
+ * @bytes: data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset,
+			   void *pdata, unsigned int bytes, bool is_read)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_mmio_info *mmio_info;
+	struct gvt_mmio_block *mmio_block;
+	gvt_mmio_func func;
+	int ret;
+
+	if (WARN_ON(bytes > 4))
+		return -EINVAL;
+
+	/*
+	 * Handle special MMIO blocks.
+	 */
+	mmio_block = find_mmio_block(gvt, offset);
+	if (mmio_block) {
+		func = is_read ? mmio_block->read : mmio_block->write;
+		if (func)
+			return func(vgpu, offset, pdata, bytes);
+		goto default_rw;
+	}
+
+	/*
+	 * Normal tracked MMIOs.
+	 */
+	mmio_info = find_mmio_info(gvt, offset);
+	if (!mmio_info) {
+		if (!vgpu->mmio.disable_warn_untrack)
+			gvt_vgpu_err("untracked MMIO %08x len %d\n",
+				     offset, bytes);
+		goto default_rw;
+	}
+
+	if (is_read)
+		return mmio_info->read(vgpu, offset, pdata, bytes);
+	else {
+		u64 ro_mask = mmio_info->ro_mask;
+		u32 old_vreg = 0, old_sreg = 0;
+		u64 data = 0;
+
+		if (intel_gvt_mmio_has_mode_mask(gvt, mmio_info->offset)) {
+			old_vreg = vgpu_vreg(vgpu, offset);
+			old_sreg = vgpu_sreg(vgpu, offset);
+		}
+
+		if (likely(!ro_mask))
+			ret = mmio_info->write(vgpu, offset, pdata, bytes);
+		else if (!~ro_mask) {
+			gvt_vgpu_err("try to write RO reg %x\n", offset);
+			return 0;
+		} else {
+			/* keep the RO bits in the virtual register */
+			memcpy(&data, pdata, bytes);
+			data &= ~ro_mask;
+			data |= vgpu_vreg(vgpu, offset) & ro_mask;
+			ret = mmio_info->write(vgpu, offset, &data, bytes);
+		}
+
+		/* higher 16bits of mode ctl regs are mask bits for change */
+		if (intel_gvt_mmio_has_mode_mask(gvt, mmio_info->offset)) {
+			u32 mask = vgpu_vreg(vgpu, offset) >> 16;
+
+			vgpu_vreg(vgpu, offset) = (old_vreg & ~mask)
+					| (vgpu_vreg(vgpu, offset) & mask);
+			vgpu_sreg(vgpu, offset) = (old_sreg & ~mask)
+					| (vgpu_sreg(vgpu, offset) & mask);
+		}
+	}
+
+	return ret;
+
+default_rw:
+	return is_read ?
+		intel_vgpu_default_mmio_read(vgpu, offset, pdata, bytes) :
+		intel_vgpu_default_mmio_write(vgpu, offset, pdata, bytes);
+}
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c
index 9d6812f0957f..7a041b368f68 100644
--- a/drivers/gpu/drm/i915/gvt/interrupt.c
+++ b/drivers/gpu/drm/i915/gvt/interrupt.c
@@ -31,6 +31,7 @@
 
 #include "i915_drv.h"
 #include "gvt.h"
+#include "trace.h"
 
 /* common offset among interrupt control registers */
 #define regbase_to_isr(base)	(base)
@@ -178,8 +179,8 @@ int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu,
 	struct intel_gvt_irq_ops *ops = gvt->irq.ops;
 	u32 imr = *(u32 *)p_data;
 
-	gvt_dbg_irq("write IMR %x, new %08x, old %08x, changed %08x\n",
-		    reg, imr, vgpu_vreg(vgpu, reg), vgpu_vreg(vgpu, reg) ^ imr);
+	trace_write_ir(vgpu->id, "IMR", reg, imr, vgpu_vreg(vgpu, reg),
+		       (vgpu_vreg(vgpu, reg) ^ imr));
 
 	vgpu_vreg(vgpu, reg) = imr;
 
@@ -209,8 +210,8 @@ int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu,
 	u32 ier = *(u32 *)p_data;
 	u32 virtual_ier = vgpu_vreg(vgpu, reg);
 
-	gvt_dbg_irq("write MASTER_IRQ %x, new %08x, old %08x, changed %08x\n",
-		    reg, ier, virtual_ier, virtual_ier ^ ier);
+	trace_write_ir(vgpu->id, "MASTER_IRQ", reg, ier, virtual_ier,
+		       (virtual_ier ^ ier));
 
 	/*
 	 * GEN8_MASTER_IRQ is a special irq register,
@@ -248,8 +249,8 @@ int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu,
 	struct intel_gvt_irq_info *info;
 	u32 ier = *(u32 *)p_data;
 
-	gvt_dbg_irq("write IER %x, new %08x, old %08x, changed %08x\n",
-		    reg, ier, vgpu_vreg(vgpu, reg), vgpu_vreg(vgpu, reg) ^ ier);
+	trace_write_ir(vgpu->id, "IER", reg, ier, vgpu_vreg(vgpu, reg),
+		       (vgpu_vreg(vgpu, reg) ^ ier));
 
 	vgpu_vreg(vgpu, reg) = ier;
 
@@ -285,8 +286,8 @@ int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg,
 		iir_to_regbase(reg));
 	u32 iir = *(u32 *)p_data;
 
-	gvt_dbg_irq("write IIR %x, new %08x, old %08x, changed %08x\n",
-		    reg, iir, vgpu_vreg(vgpu, reg), vgpu_vreg(vgpu, reg) ^ iir);
+	trace_write_ir(vgpu->id, "IIR", reg, iir, vgpu_vreg(vgpu, reg),
+		       (vgpu_vreg(vgpu, reg) ^ iir));
 
 	if (WARN_ON(!info))
 		return -EINVAL;
@@ -411,8 +412,7 @@ static void propagate_event(struct intel_gvt_irq *irq,
 
 	if (!test_bit(bit, (void *)&vgpu_vreg(vgpu,
 					regbase_to_imr(reg_base)))) {
-		gvt_dbg_irq("set bit (%d) for (%s) for vgpu (%d)\n",
-				bit, irq_name[event], vgpu->id);
+		trace_propagate_event(vgpu->id, irq_name[event], bit);
 		set_bit(bit, (void *)&vgpu_vreg(vgpu,
 					regbase_to_iir(reg_base)));
 	}
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 1ba3bdb09341..980ec8906b1e 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -123,7 +123,6 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
 		void *p_data, unsigned int bytes)
 {
 	struct intel_gvt *gvt = vgpu->gvt;
-	struct intel_gvt_mmio_info *mmio;
 	unsigned int offset = 0;
 	int ret = -EINVAL;
 
@@ -187,32 +186,8 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
 			goto err;
 	}
 
-	mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
-	if (mmio) {
-		if (!intel_gvt_mmio_is_unalign(gvt, mmio->offset)) {
-			if (WARN_ON(offset + bytes > mmio->offset + mmio->size))
-				goto err;
-			if (WARN_ON(mmio->offset != offset))
-				goto err;
-		}
-		ret = mmio->read(vgpu, offset, p_data, bytes);
-	} else {
-		ret = intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
-
-		if (!vgpu->mmio.disable_warn_untrack) {
-			gvt_vgpu_err("read untracked MMIO %x(%dB) val %x\n",
-				offset, bytes, *(u32 *)p_data);
-
-			if (offset == 0x206c) {
-				gvt_vgpu_err("------------------------------------------\n");
-				gvt_vgpu_err("likely triggers a gfx reset\n");
-				gvt_vgpu_err("------------------------------------------\n");
-				vgpu->mmio.disable_warn_untrack = true;
-			}
-		}
-	}
-
-	if (ret)
+	ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, true);
+	if (ret < 0)
 		goto err;
 
 	intel_gvt_mmio_set_accessed(gvt, offset);
@@ -239,9 +214,7 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
 		void *p_data, unsigned int bytes)
 {
 	struct intel_gvt *gvt = vgpu->gvt;
-	struct intel_gvt_mmio_info *mmio;
 	unsigned int offset = 0;
-	u32 old_vreg = 0, old_sreg = 0;
 	int ret = -EINVAL;
 
 	if (vgpu->failsafe) {
@@ -296,66 +269,10 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
 		return ret;
 	}
 
-	mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
-	if (!mmio && !vgpu->mmio.disable_warn_untrack)
-		gvt_dbg_mmio("vgpu%d: write untracked MMIO %x len %d val %x\n",
-				vgpu->id, offset, bytes, *(u32 *)p_data);
-
-	if (!intel_gvt_mmio_is_unalign(gvt, offset)) {
-		if (WARN_ON(!IS_ALIGNED(offset, bytes)))
-			goto err;
-	}
-
-	if (mmio) {
-		u64 ro_mask = mmio->ro_mask;
-
-		if (!intel_gvt_mmio_is_unalign(gvt, mmio->offset)) {
-			if (WARN_ON(offset + bytes > mmio->offset + mmio->size))
-				goto err;
-			if (WARN_ON(mmio->offset != offset))
-				goto err;
-		}
-
-		if (intel_gvt_mmio_has_mode_mask(gvt, mmio->offset)) {
-			old_vreg = vgpu_vreg(vgpu, offset);
-			old_sreg = vgpu_sreg(vgpu, offset);
-		}
-
-		if (!ro_mask) {
-			ret = mmio->write(vgpu, offset, p_data, bytes);
-		} else {
-			/* Protect RO bits like HW */
-			u64 data = 0;
-
-			/* all register bits are RO. */
-			if (ro_mask == ~(u64)0) {
-				gvt_vgpu_err("try to write RO reg %x\n",
-					offset);
-				ret = 0;
-				goto out;
-			}
-			/* keep the RO bits in the virtual register */
-			memcpy(&data, p_data, bytes);
-			data &= ~mmio->ro_mask;
-			data |= vgpu_vreg(vgpu, offset) & mmio->ro_mask;
-			ret = mmio->write(vgpu, offset, &data, bytes);
-		}
-
-		/* higher 16bits of mode ctl regs are mask bits for change */
-		if (intel_gvt_mmio_has_mode_mask(gvt, mmio->offset)) {
-			u32 mask = vgpu_vreg(vgpu, offset) >> 16;
-
-			vgpu_vreg(vgpu, offset) = (old_vreg & ~mask)
-				| (vgpu_vreg(vgpu, offset) & mask);
-			vgpu_sreg(vgpu, offset) = (old_sreg & ~mask)
-				| (vgpu_sreg(vgpu, offset) & mask);
-		}
-	} else
-		ret = intel_vgpu_default_mmio_write(vgpu, offset, p_data,
-				bytes);
-	if (ret)
+	ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, false);
+	if (ret < 0)
 		goto err;
-out:
+
 	intel_gvt_mmio_set_accessed(gvt, offset);
 	mutex_unlock(&gvt->lock);
 	return 0;
@@ -372,20 +289,32 @@ err:
  * @vgpu: a vGPU
  *
  */
-void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu)
+void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr)
 {
 	struct intel_gvt *gvt = vgpu->gvt;
 	const struct intel_gvt_device_info *info = &gvt->device_info;
+	void  *mmio = gvt->firmware.mmio;
+
+	if (dmlr) {
+		memcpy(vgpu->mmio.vreg, mmio, info->mmio_size);
+		memcpy(vgpu->mmio.sreg, mmio, info->mmio_size);
 
-	memcpy(vgpu->mmio.vreg, gvt->firmware.mmio, info->mmio_size);
-	memcpy(vgpu->mmio.sreg, gvt->firmware.mmio, info->mmio_size);
+		vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
 
-	vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
+		/* set the bit 0:2(Core C-State ) to C0 */
+		vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0;
 
-	/* set the bit 0:2(Core C-State ) to C0 */
-	vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0;
+		vgpu->mmio.disable_warn_untrack = false;
+	} else {
+#define GVT_GEN8_MMIO_RESET_OFFSET		(0x44200)
+		/* only reset the engine related, so starting with 0x44200
+		 * interrupt include DE,display mmio related will not be
+		 * touched
+		 */
+		memcpy(vgpu->mmio.vreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET);
+		memcpy(vgpu->mmio.sreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET);
+	}
 
-	vgpu->mmio.disable_warn_untrack = false;
 }
 
 /**
@@ -405,7 +334,7 @@ int intel_vgpu_init_mmio(struct intel_vgpu *vgpu)
 
 	vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size;
 
-	intel_vgpu_reset_mmio(vgpu);
+	intel_vgpu_reset_mmio(vgpu, true);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
index 7edd66f38ef9..32cd64ddad26 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.h
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -39,36 +39,28 @@
 struct intel_gvt;
 struct intel_vgpu;
 
-#define D_SNB   (1 << 0)
-#define D_IVB   (1 << 1)
-#define D_HSW   (1 << 2)
-#define D_BDW   (1 << 3)
-#define D_SKL	(1 << 4)
-#define D_KBL	(1 << 5)
+#define D_BDW   (1 << 0)
+#define D_SKL	(1 << 1)
+#define D_KBL	(1 << 2)
 
 #define D_GEN9PLUS	(D_SKL | D_KBL)
 #define D_GEN8PLUS	(D_BDW | D_SKL | D_KBL)
-#define D_GEN75PLUS	(D_HSW | D_BDW | D_SKL | D_KBL)
-#define D_GEN7PLUS	(D_IVB | D_HSW | D_BDW | D_SKL | D_KBL)
 
 #define D_SKL_PLUS	(D_SKL | D_KBL)
 #define D_BDW_PLUS	(D_BDW | D_SKL | D_KBL)
-#define D_HSW_PLUS	(D_HSW | D_BDW | D_SKL | D_KBL)
-#define D_IVB_PLUS	(D_IVB | D_HSW | D_BDW | D_SKL | D_KBL)
 
-#define D_PRE_BDW	(D_SNB | D_IVB | D_HSW)
-#define D_PRE_SKL	(D_SNB | D_IVB | D_HSW | D_BDW)
-#define D_ALL		(D_SNB | D_IVB | D_HSW | D_BDW | D_SKL | D_KBL)
+#define D_PRE_SKL	(D_BDW)
+#define D_ALL		(D_BDW | D_SKL | D_KBL)
+
+typedef int (*gvt_mmio_func)(struct intel_vgpu *, unsigned int, void *,
+			     unsigned int);
 
 struct intel_gvt_mmio_info {
 	u32 offset;
-	u32 size;
-	u32 length;
-	u32 addr_mask;
 	u64 ro_mask;
 	u32 device;
-	int (*read)(struct intel_vgpu *, unsigned int, void *, unsigned int);
-	int (*write)(struct intel_vgpu *, unsigned int, void *, unsigned int);
+	gvt_mmio_func read;
+	gvt_mmio_func write;
 	u32 addr_range;
 	struct hlist_node node;
 };
@@ -79,8 +71,6 @@ bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device);
 int intel_gvt_setup_mmio_info(struct intel_gvt *gvt);
 void intel_gvt_clean_mmio_info(struct intel_gvt *gvt);
 
-struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
-						     unsigned int offset);
 #define INTEL_GVT_MMIO_OFFSET(reg) ({ \
 	typeof(reg) __reg = reg; \
 	u32 *offset = (u32 *)&__reg; \
@@ -88,7 +78,7 @@ struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
 })
 
 int intel_vgpu_init_mmio(struct intel_vgpu *vgpu);
-void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu);
+void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr);
 void intel_vgpu_clean_mmio(struct intel_vgpu *vgpu);
 
 int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa);
@@ -97,13 +87,7 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa,
 				void *p_data, unsigned int bytes);
 int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa,
 				void *p_data, unsigned int bytes);
-bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
-				  unsigned int offset);
-bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt, unsigned int offset);
-void intel_gvt_mmio_set_accessed(struct intel_gvt *gvt, unsigned int offset);
-void intel_gvt_mmio_set_cmd_accessed(struct intel_gvt *gvt,
-				     unsigned int offset);
-bool intel_gvt_mmio_has_mode_mask(struct intel_gvt *gvt, unsigned int offset);
+
 int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
 				 void *p_data, unsigned int bytes);
 int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
@@ -111,4 +95,8 @@ int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 
 bool intel_gvt_in_force_nonpriv_whitelist(struct intel_gvt *gvt,
 					  unsigned int offset);
+
+int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset,
+			   void *pdata, unsigned int bytes, bool is_read);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index 419353624c5a..f0e5487e6688 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -133,8 +133,7 @@ static inline int intel_gvt_hypervisor_inject_msi(struct intel_vgpu *vgpu)
 	if (WARN(control & GENMASK(15, 1), "only support one MSI format\n"))
 		return -EINVAL;
 
-	gvt_dbg_irq("vgpu%d: inject msi address %x data%x\n", vgpu->id, addr,
-		    data);
+	trace_inject_msi(vgpu->id, addr, data);
 
 	ret = intel_gvt_host.mpt->inject_msi(vgpu->handle, addr, data);
 	if (ret)
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
index a5e11d89df2f..504e57c3bc23 100644
--- a/drivers/gpu/drm/i915/gvt/render.c
+++ b/drivers/gpu/drm/i915/gvt/render.c
@@ -35,6 +35,7 @@
 
 #include "i915_drv.h"
 #include "gvt.h"
+#include "trace.h"
 
 struct render_mmio {
 	int ring_id;
@@ -260,7 +261,8 @@ static void restore_mocs(struct intel_vgpu *vgpu, int ring_id)
 
 #define CTX_CONTEXT_CONTROL_VAL	0x03
 
-void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id)
+/* Switch ring mmio values (context) from host to a vgpu. */
+static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct render_mmio *mmio;
@@ -305,14 +307,15 @@ void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id)
 		I915_WRITE(mmio->reg, v);
 		POSTING_READ(mmio->reg);
 
-		gvt_dbg_render("load reg %x old %x new %x\n",
-				i915_mmio_reg_offset(mmio->reg),
-				mmio->value, v);
+		trace_render_mmio(vgpu->id, "load",
+				  i915_mmio_reg_offset(mmio->reg),
+				  mmio->value, v);
 	}
 	handle_tlb_pending_event(vgpu, ring_id);
 }
 
-void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id)
+/* Switch ring mmio values (context) from vgpu to host. */
+static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct render_mmio *mmio;
@@ -346,8 +349,37 @@ void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id)
 		I915_WRITE(mmio->reg, v);
 		POSTING_READ(mmio->reg);
 
-		gvt_dbg_render("restore reg %x old %x new %x\n",
-				i915_mmio_reg_offset(mmio->reg),
-				mmio->value, v);
+		trace_render_mmio(vgpu->id, "restore",
+				  i915_mmio_reg_offset(mmio->reg),
+				  mmio->value, v);
 	}
 }
+
+/**
+ * intel_gvt_switch_render_mmio - switch mmio context of specific engine
+ * @pre: the last vGPU that own the engine
+ * @next: the vGPU to switch to
+ * @ring_id: specify the engine
+ *
+ * If pre is null indicates that host own the engine. If next is null
+ * indicates that we are switching to host workload.
+ */
+void intel_gvt_switch_mmio(struct intel_vgpu *pre,
+			   struct intel_vgpu *next, int ring_id)
+{
+	if (WARN_ON(!pre && !next))
+		return;
+
+	gvt_dbg_render("switch ring %d from %s to %s\n", ring_id,
+		       pre ? "vGPU" : "host", next ? "vGPU" : "HOST");
+
+	/**
+	 * TODO: Optimize for vGPU to vGPU switch by merging
+	 * switch_mmio_to_host() and switch_mmio_to_vgpu().
+	 */
+	if (pre)
+		switch_mmio_to_host(pre, ring_id);
+
+	if (next)
+		switch_mmio_to_vgpu(next, ring_id);
+}
diff --git a/drivers/gpu/drm/i915/gvt/render.h b/drivers/gpu/drm/i915/gvt/render.h
index dac1a3cc458b..91db1d39d28f 100644
--- a/drivers/gpu/drm/i915/gvt/render.h
+++ b/drivers/gpu/drm/i915/gvt/render.h
@@ -36,8 +36,8 @@
 #ifndef __GVT_RENDER_H__
 #define __GVT_RENDER_H__
 
-void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id);
+void intel_gvt_switch_mmio(struct intel_vgpu *pre,
+			   struct intel_vgpu *next, int ring_id);
 
-void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
index f25ff133865f..436377da41ba 100644
--- a/drivers/gpu/drm/i915/gvt/sched_policy.c
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -202,11 +202,6 @@ static void tbs_sched_func(struct gvt_sched_data *sched_data)
 	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 	struct vgpu_sched_data *vgpu_data;
 	struct intel_vgpu *vgpu = NULL;
-	static uint64_t timer_check;
-
-	if (!(timer_check++ % GVT_TS_BALANCE_PERIOD_MS))
-		gvt_balance_timeslice(sched_data);
-
 	/* no active vgpu or has already had a target */
 	if (list_empty(&sched_data->lru_runq_head) || scheduler->next_vgpu)
 		goto out;
@@ -231,9 +226,19 @@ out:
 void intel_gvt_schedule(struct intel_gvt *gvt)
 {
 	struct gvt_sched_data *sched_data = gvt->scheduler.sched_data;
+	static uint64_t timer_check;
 
 	mutex_lock(&gvt->lock);
+
+	if (test_and_clear_bit(INTEL_GVT_REQUEST_SCHED,
+				(void *)&gvt->service_request)) {
+		if (!(timer_check++ % GVT_TS_BALANCE_PERIOD_MS))
+			gvt_balance_timeslice(sched_data);
+	}
+	clear_bit(INTEL_GVT_REQUEST_EVENT_SCHED, (void *)&gvt->service_request);
+
 	tbs_sched_func(sched_data);
+
 	mutex_unlock(&gvt->lock);
 }
 
@@ -303,8 +308,20 @@ static int tbs_sched_init_vgpu(struct intel_vgpu *vgpu)
 
 static void tbs_sched_clean_vgpu(struct intel_vgpu *vgpu)
 {
+	struct intel_gvt_workload_scheduler *scheduler = &vgpu->gvt->scheduler;
+	int ring_id;
+
 	kfree(vgpu->sched_data);
 	vgpu->sched_data = NULL;
+
+	spin_lock_bh(&scheduler->mmio_context_lock);
+	for (ring_id = 0; ring_id < I915_NUM_ENGINES; ring_id++) {
+		if (scheduler->engine_owner[ring_id] == vgpu) {
+			intel_gvt_switch_mmio(vgpu, NULL, ring_id);
+			scheduler->engine_owner[ring_id] = NULL;
+		}
+	}
+	spin_unlock_bh(&scheduler->mmio_context_lock);
 }
 
 static void tbs_sched_start_schedule(struct intel_vgpu *vgpu)
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index bada32b33237..488fdea348a9 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -69,8 +69,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 	gvt_dbg_sched("ring id %d workload lrca %x", ring_id,
 			workload->ctx_desc.lrca);
 
-	context_page_num = intel_lr_context_size(
-			gvt->dev_priv->engine[ring_id]);
+	context_page_num = gvt->dev_priv->engine[ring_id]->context_size;
 
 	context_page_num = context_page_num >> PAGE_SHIFT;
 
@@ -139,21 +138,42 @@ static int shadow_context_status_change(struct notifier_block *nb,
 	struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
 				shadow_ctx_notifier_block[req->engine->id]);
 	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
-	struct intel_vgpu_workload *workload =
-		scheduler->current_workload[req->engine->id];
+	enum intel_engine_id ring_id = req->engine->id;
+	struct intel_vgpu_workload *workload;
+
+	if (!is_gvt_request(req)) {
+		spin_lock_bh(&scheduler->mmio_context_lock);
+		if (action == INTEL_CONTEXT_SCHEDULE_IN &&
+		    scheduler->engine_owner[ring_id]) {
+			/* Switch ring from vGPU to host. */
+			intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
+					      NULL, ring_id);
+			scheduler->engine_owner[ring_id] = NULL;
+		}
+		spin_unlock_bh(&scheduler->mmio_context_lock);
 
-	if (!is_gvt_request(req) || unlikely(!workload))
+		return NOTIFY_OK;
+	}
+
+	workload = scheduler->current_workload[ring_id];
+	if (unlikely(!workload))
 		return NOTIFY_OK;
 
 	switch (action) {
 	case INTEL_CONTEXT_SCHEDULE_IN:
-		intel_gvt_load_render_mmio(workload->vgpu,
-					   workload->ring_id);
+		spin_lock_bh(&scheduler->mmio_context_lock);
+		if (workload->vgpu != scheduler->engine_owner[ring_id]) {
+			/* Switch ring from host to vGPU or vGPU to vGPU. */
+			intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
+					      workload->vgpu, ring_id);
+			scheduler->engine_owner[ring_id] = workload->vgpu;
+		} else
+			gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n",
+				      ring_id, workload->vgpu->id);
+		spin_unlock_bh(&scheduler->mmio_context_lock);
 		atomic_set(&workload->shadow_ctx_active, 1);
 		break;
 	case INTEL_CONTEXT_SCHEDULE_OUT:
-		intel_gvt_restore_render_mmio(workload->vgpu,
-					      workload->ring_id);
 		/* If the status is -EINPROGRESS means this workload
 		 * doesn't meet any issue during dispatching so when
 		 * get the SCHEDULE_OUT set the status to be zero for
@@ -181,6 +201,7 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 	struct intel_engine_cs *engine = dev_priv->engine[ring_id];
 	struct drm_i915_gem_request *rq;
 	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_ring *ring;
 	int ret;
 
 	gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
@@ -199,8 +220,9 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 	 * shadow_ctx pages invalid. So gvt need to pin itself. After update
 	 * the guest context, gvt can unpin the shadow_ctx safely.
 	 */
-	ret = engine->context_pin(engine, shadow_ctx);
-	if (ret) {
+	ring = engine->context_pin(engine, shadow_ctx);
+	if (IS_ERR(ring)) {
+		ret = PTR_ERR(ring);
 		gvt_vgpu_err("fail to pin shadow context\n");
 		workload->status = ret;
 		mutex_unlock(&dev_priv->drm.struct_mutex);
@@ -330,8 +352,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 	gvt_dbg_sched("ring id %d workload lrca %x\n", ring_id,
 			workload->ctx_desc.lrca);
 
-	context_page_num = intel_lr_context_size(
-			gvt->dev_priv->engine[ring_id]);
+	context_page_num = gvt->dev_priv->engine[ring_id]->context_size;
 
 	context_page_num = context_page_num >> PAGE_SHIFT;
 
@@ -431,6 +452,10 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 
 	atomic_dec(&vgpu->running_workload_num);
 	wake_up(&scheduler->workload_complete_wq);
+
+	if (gvt->scheduler.need_reschedule)
+		intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);
+
 	mutex_unlock(&gvt->lock);
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index 2cd725c0573e..9b6bf51e9b9b 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -42,6 +42,10 @@ struct intel_gvt_workload_scheduler {
 	struct intel_vgpu_workload *current_workload[I915_NUM_ENGINES];
 	bool need_reschedule;
 
+	spinlock_t mmio_context_lock;
+	/* can be null when owner is host */
+	struct intel_vgpu *engine_owner[I915_NUM_ENGINES];
+
 	wait_queue_head_t workload_complete_wq;
 	struct task_struct *thread[I915_NUM_ENGINES];
 	wait_queue_head_t waitq[I915_NUM_ENGINES];
diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h
index 53a2d10cf3f1..8c150381d9a4 100644
--- a/drivers/gpu/drm/i915/gvt/trace.h
+++ b/drivers/gpu/drm/i915/gvt/trace.h
@@ -224,58 +224,138 @@ TRACE_EVENT(oos_sync,
 	TP_printk("%s", __entry->buf)
 );
 
-#define MAX_CMD_STR_LEN	256
 TRACE_EVENT(gvt_command,
-		TP_PROTO(u8 vm_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len, bool ring_buffer_cmd, cycles_t cost_pre_cmd_handler, cycles_t cost_cmd_handler),
-
-		TP_ARGS(vm_id, ring_id, ip_gma, cmd_va, cmd_len, ring_buffer_cmd, cost_pre_cmd_handler, cost_cmd_handler),
-
-		TP_STRUCT__entry(
-			__field(u8, vm_id)
-			__field(u8, ring_id)
-			__field(int, i)
-			__array(char, tmp_buf, MAX_CMD_STR_LEN)
-			__array(char, cmd_str, MAX_CMD_STR_LEN)
-			),
-
-		TP_fast_assign(
-			__entry->vm_id = vm_id;
-			__entry->ring_id = ring_id;
-			__entry->cmd_str[0] = '\0';
-			snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "VM(%d) Ring(%d): %s ip(%08x) pre handler cost (%llu), handler cost (%llu) ", vm_id, ring_id, ring_buffer_cmd ? "RB":"BB", ip_gma, cost_pre_cmd_handler, cost_cmd_handler);
-			strcat(__entry->cmd_str, __entry->tmp_buf);
-			entry->i = 0;
-			while (cmd_len > 0) {
-				if (cmd_len >= 8) {
-					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x %08x %08x %08x %08x %08x %08x ",
-						cmd_va[__entry->i], cmd_va[__entry->i+1], cmd_va[__entry->i+2], cmd_va[__entry->i+3],
-						cmd_va[__entry->i+4], cmd_va[__entry->i+5], cmd_va[__entry->i+6], cmd_va[__entry->i+7]);
-					__entry->i += 8;
-					cmd_len -= 8;
-					strcat(__entry->cmd_str, __entry->tmp_buf);
-				} else if (cmd_len >= 4) {
-					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x %08x %08x ",
-						cmd_va[__entry->i], cmd_va[__entry->i+1], cmd_va[__entry->i+2], cmd_va[__entry->i+3]);
-					__entry->i += 4;
-					cmd_len -= 4;
-					strcat(__entry->cmd_str, __entry->tmp_buf);
-				} else if (cmd_len >= 2) {
-					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x ", cmd_va[__entry->i], cmd_va[__entry->i+1]);
-					__entry->i += 2;
-					cmd_len -= 2;
-					strcat(__entry->cmd_str, __entry->tmp_buf);
-				} else if (cmd_len == 1) {
-					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x ", cmd_va[__entry->i]);
-					__entry->i += 1;
-					cmd_len -= 1;
-					strcat(__entry->cmd_str, __entry->tmp_buf);
-				}
-			}
-			strcat(__entry->cmd_str, "\n");
-		),
+	TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len,
+		 u32 buf_type),
+
+	TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type),
+
+	TP_STRUCT__entry(
+		__field(u8, vgpu_id)
+		__field(u8, ring_id)
+		__field(u32, ip_gma)
+		__field(u32, buf_type)
+		__field(u32, cmd_len)
+		__dynamic_array(u32, raw_cmd, cmd_len)
+	),
+
+	TP_fast_assign(
+		__entry->vgpu_id = vgpu_id;
+		__entry->ring_id = ring_id;
+		__entry->ip_gma = ip_gma;
+		__entry->buf_type = buf_type;
+		__entry->cmd_len = cmd_len;
+		memcpy(__get_dynamic_array(raw_cmd), cmd_va, cmd_len * sizeof(*cmd_va));
+	),
+
+
+	TP_printk("vgpu%d ring %d: buf_type %u, ip_gma %08x, raw cmd %s",
+		__entry->vgpu_id,
+		__entry->ring_id,
+		__entry->buf_type,
+		__entry->ip_gma,
+		__print_array(__get_dynamic_array(raw_cmd), __entry->cmd_len, 4))
+);
+
+#define GVT_TEMP_STR_LEN 10
+TRACE_EVENT(write_ir,
+	TP_PROTO(int id, char *reg_name, unsigned int reg, unsigned int new_val,
+		 unsigned int old_val, bool changed),
+
+	TP_ARGS(id, reg_name, reg, new_val, old_val, changed),
+
+	TP_STRUCT__entry(
+		__field(int, id)
+		__array(char, buf, GVT_TEMP_STR_LEN)
+		__field(unsigned int, reg)
+		__field(unsigned int, new_val)
+		__field(unsigned int, old_val)
+		__field(bool, changed)
+	),
+
+	TP_fast_assign(
+		__entry->id = id;
+		snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", reg_name);
+		__entry->reg = reg;
+		__entry->new_val = new_val;
+		__entry->old_val = old_val;
+		__entry->changed = changed;
+	),
+
+	TP_printk("VM%u write [%s] %x, new %08x, old %08x, changed %08x\n",
+		  __entry->id, __entry->buf, __entry->reg, __entry->new_val,
+		  __entry->old_val, __entry->changed)
+);
+
+TRACE_EVENT(propagate_event,
+	TP_PROTO(int id, const char *irq_name, int bit),
+
+	TP_ARGS(id, irq_name, bit),
+
+	TP_STRUCT__entry(
+		__field(int, id)
+		__array(char, buf, GVT_TEMP_STR_LEN)
+		__field(int, bit)
+	),
 
-		TP_printk("%s", __entry->cmd_str)
+	TP_fast_assign(
+		__entry->id = id;
+		snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", irq_name);
+		__entry->bit = bit;
+	),
+
+	TP_printk("Set bit (%d) for (%s) for vgpu (%d)\n",
+		  __entry->bit, __entry->buf, __entry->id)
 );
+
+TRACE_EVENT(inject_msi,
+	TP_PROTO(int id, unsigned int address, unsigned int data),
+
+	TP_ARGS(id, address, data),
+
+	TP_STRUCT__entry(
+		__field(int, id)
+		__field(unsigned int, address)
+		__field(unsigned int, data)
+	),
+
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->address = address;
+		__entry->data = data;
+	),
+
+	TP_printk("vgpu%d:inject msi address %x data %x\n",
+		  __entry->id, __entry->address, __entry->data)
+);
+
+TRACE_EVENT(render_mmio,
+	TP_PROTO(int id, char *action, unsigned int reg,
+		 unsigned int old_val, unsigned int new_val),
+
+	TP_ARGS(id, action, reg, new_val, old_val),
+
+	TP_STRUCT__entry(
+		__field(int, id)
+		__array(char, buf, GVT_TEMP_STR_LEN)
+		__field(unsigned int, reg)
+		__field(unsigned int, old_val)
+		__field(unsigned int, new_val)
+	),
+
+	TP_fast_assign(
+		__entry->id = id;
+		snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", action);
+		__entry->reg = reg;
+		__entry->old_val = old_val;
+		__entry->new_val = new_val;
+	),
+
+	TP_printk("VM%u %s reg %x, old %08x new %08x\n",
+		  __entry->id, __entry->buf, __entry->reg,
+		  __entry->old_val, __entry->new_val)
+);
+
 #endif /* _GVT_TRACE_H_ */
 
 /* This part must be out of protection */
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 6e3cbd8caec2..90c14e6e3ea0 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -501,9 +501,14 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
 
 	/* full GPU reset or device model level reset */
 	if (engine_mask == ALL_ENGINES || dmlr) {
+
 		intel_vgpu_reset_gtt(vgpu, dmlr);
-		intel_vgpu_reset_resource(vgpu);
-		intel_vgpu_reset_mmio(vgpu);
+
+		/*fence will not be reset during virtual reset */
+		if (dmlr)
+			intel_vgpu_reset_resource(vgpu);
+
+		intel_vgpu_reset_mmio(vgpu, dmlr);
 		populate_pvinfo_page(vgpu);
 		intel_vgpu_reset_display(vgpu);
 
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 7af100f84410..f0cb22cc0dd6 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -1166,8 +1166,8 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 				find_reg(engine, is_master, reg_addr);
 
 			if (!reg) {
-				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (exec_id=%d)\n",
-						 reg_addr, *cmd, engine->exec_id);
+				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n",
+						 reg_addr, *cmd, engine->name);
 				return false;
 			}
 
@@ -1222,11 +1222,11 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 				desc->bits[i].mask;
 
 			if (dword != desc->bits[i].expected) {
-				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (exec_id=%d)\n",
+				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n",
 						 *cmd,
 						 desc->bits[i].mask,
 						 desc->bits[i].expected,
-						 dword, engine->exec_id);
+						 dword, engine->name);
 				return false;
 			}
 		}
@@ -1284,7 +1284,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
 
 		if (*cmd == MI_BATCH_BUFFER_END) {
 			if (needs_clflush_after) {
-				void *ptr = ptr_mask_bits(shadow_batch_obj->mm.mapping);
+				void *ptr = page_mask_bits(shadow_batch_obj->mm.mapping);
 				drm_clflush_virt_range(ptr,
 						       (void *)(cmd + 1) - ptr);
 			}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 4bd1467c17b1..3f44076ec8a0 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -229,7 +229,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 	int ret;
 
 	total = READ_ONCE(dev_priv->mm.object_count);
-	objects = drm_malloc_ab(total, sizeof(*objects));
+	objects = kvmalloc_array(total, sizeof(*objects), GFP_KERNEL);
 	if (!objects)
 		return -ENOMEM;
 
@@ -274,7 +274,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 
 	mutex_unlock(&dev->struct_mutex);
 out:
-	drm_free_large(objects);
+	kvfree(objects);
 	return ret;
 }
 
@@ -1674,12 +1674,22 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 		seq_printf(m, "FBC disabled: %s\n",
 			   dev_priv->fbc.no_fbc_reason);
 
-	if (intel_fbc_is_active(dev_priv) && INTEL_GEN(dev_priv) >= 7) {
-		uint32_t mask = INTEL_GEN(dev_priv) >= 8 ?
-				BDW_FBC_COMPRESSION_MASK :
-				IVB_FBC_COMPRESSION_MASK;
-		seq_printf(m, "Compressing: %s\n",
-			   yesno(I915_READ(FBC_STATUS2) & mask));
+	if (intel_fbc_is_active(dev_priv)) {
+		u32 mask;
+
+		if (INTEL_GEN(dev_priv) >= 8)
+			mask = I915_READ(IVB_FBC_STATUS2) & BDW_FBC_COMP_SEG_MASK;
+		else if (INTEL_GEN(dev_priv) >= 7)
+			mask = I915_READ(IVB_FBC_STATUS2) & IVB_FBC_COMP_SEG_MASK;
+		else if (INTEL_GEN(dev_priv) >= 5)
+			mask = I915_READ(ILK_DPFC_STATUS) & ILK_DPFC_COMP_SEG_MASK;
+		else if (IS_G4X(dev_priv))
+			mask = I915_READ(DPFC_STATUS) & DPFC_COMP_SEG_MASK;
+		else
+			mask = I915_READ(FBC_STATUS) & (FBC_STAT_COMPRESSING |
+							FBC_STAT_COMPRESSED);
+
+		seq_printf(m, "Compressing: %s\n", yesno(mask));
 	}
 
 	mutex_unlock(&dev_priv->fbc.lock);
@@ -1688,7 +1698,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static int i915_fbc_fc_get(void *data, u64 *val)
+static int i915_fbc_false_color_get(void *data, u64 *val)
 {
 	struct drm_i915_private *dev_priv = data;
 
@@ -1700,7 +1710,7 @@ static int i915_fbc_fc_get(void *data, u64 *val)
 	return 0;
 }
 
-static int i915_fbc_fc_set(void *data, u64 val)
+static int i915_fbc_false_color_set(void *data, u64 val)
 {
 	struct drm_i915_private *dev_priv = data;
 	u32 reg;
@@ -1721,8 +1731,8 @@ static int i915_fbc_fc_set(void *data, u64 val)
 	return 0;
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_fc_fops,
-			i915_fbc_fc_get, i915_fbc_fc_set,
+DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_false_color_fops,
+			i915_fbc_false_color_get, i915_fbc_false_color_set,
 			"%llu\n");
 
 static int i915_ips_status(struct seq_file *m, void *unused)
@@ -1992,6 +2002,12 @@ static int i915_context_status(struct seq_file *m, void *unused)
 			seq_putc(m, '\n');
 		}
 
+		seq_printf(m,
+			   "\tvma hashtable size=%u (actual %lu), count=%u\n",
+			   ctx->vma_lut.ht_size,
+			   BIT(ctx->vma_lut.ht_bits),
+			   ctx->vma_lut.ht_count);
+
 		seq_putc(m, '\n');
 	}
 
@@ -2486,8 +2502,6 @@ static void i915_guc_client_info(struct seq_file *m,
 		client->wq_size, client->wq_offset, client->wq_tail);
 
 	seq_printf(m, "\tWork queue full: %u\n", client->no_wq_space);
-	seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail);
-	seq_printf(m, "\tLast submission result: %d\n", client->retcode);
 
 	for_each_engine(engine, dev_priv, id) {
 		u64 submissions = client->submissions[id];
@@ -2498,42 +2512,34 @@ static void i915_guc_client_info(struct seq_file *m,
 	seq_printf(m, "\tTotal: %llu\n", tot);
 }
 
-static int i915_guc_info(struct seq_file *m, void *data)
+static bool check_guc_submission(struct seq_file *m)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	const struct intel_guc *guc = &dev_priv->guc;
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
-	u64 total;
 
 	if (!guc->execbuf_client) {
 		seq_printf(m, "GuC submission %s\n",
 			   HAS_GUC_SCHED(dev_priv) ?
 			   "disabled" :
 			   "not supported");
-		return 0;
+		return false;
 	}
 
+	return true;
+}
+
+static int i915_guc_info(struct seq_file *m, void *data)
+{
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	const struct intel_guc *guc = &dev_priv->guc;
+
+	if (!check_guc_submission(m))
+		return 0;
+
 	seq_printf(m, "Doorbell map:\n");
 	seq_printf(m, "\t%*pb\n", GUC_NUM_DOORBELLS, guc->doorbell_bitmap);
 	seq_printf(m, "Doorbell next cacheline: 0x%x\n\n", guc->db_cacheline);
 
-	seq_printf(m, "GuC total action count: %llu\n", guc->action_count);
-	seq_printf(m, "GuC action failure count: %u\n", guc->action_fail);
-	seq_printf(m, "GuC last action command: 0x%x\n", guc->action_cmd);
-	seq_printf(m, "GuC last action status: 0x%x\n", guc->action_status);
-	seq_printf(m, "GuC last action error code: %d\n", guc->action_err);
-
-	total = 0;
-	seq_printf(m, "\nGuC submissions:\n");
-	for_each_engine(engine, dev_priv, id) {
-		u64 submissions = guc->submissions[id];
-		total += submissions;
-		seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x\n",
-			engine->name, submissions, guc->last_seqno[id]);
-	}
-	seq_printf(m, "\t%s: %llu\n", "Total", total);
-
 	seq_printf(m, "\nGuC execbuf client @ %p:\n", guc->execbuf_client);
 	i915_guc_client_info(m, dev_priv, guc->execbuf_client);
 
@@ -2544,36 +2550,99 @@ static int i915_guc_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static int i915_guc_log_dump(struct seq_file *m, void *data)
+static int i915_guc_stage_pool(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_i915_gem_object *obj;
-	int i = 0, pg;
+	const struct intel_guc *guc = &dev_priv->guc;
+	struct guc_stage_desc *desc = guc->stage_desc_pool_vaddr;
+	struct i915_guc_client *client = guc->execbuf_client;
+	unsigned int tmp;
+	int index;
 
-	if (!dev_priv->guc.log.vma)
+	if (!check_guc_submission(m))
 		return 0;
 
-	obj = dev_priv->guc.log.vma->obj;
-	for (pg = 0; pg < obj->base.size / PAGE_SIZE; pg++) {
-		u32 *log = kmap_atomic(i915_gem_object_get_page(obj, pg));
+	for (index = 0; index < GUC_MAX_STAGE_DESCRIPTORS; index++, desc++) {
+		struct intel_engine_cs *engine;
+
+		if (!(desc->attribute & GUC_STAGE_DESC_ATTR_ACTIVE))
+			continue;
 
-		for (i = 0; i < PAGE_SIZE / sizeof(u32); i += 4)
-			seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n",
-				   *(log + i), *(log + i + 1),
-				   *(log + i + 2), *(log + i + 3));
+		seq_printf(m, "GuC stage descriptor %u:\n", index);
+		seq_printf(m, "\tIndex: %u\n", desc->stage_id);
+		seq_printf(m, "\tAttribute: 0x%x\n", desc->attribute);
+		seq_printf(m, "\tPriority: %d\n", desc->priority);
+		seq_printf(m, "\tDoorbell id: %d\n", desc->db_id);
+		seq_printf(m, "\tEngines used: 0x%x\n",
+			   desc->engines_used);
+		seq_printf(m, "\tDoorbell trigger phy: 0x%llx, cpu: 0x%llx, uK: 0x%x\n",
+			   desc->db_trigger_phy,
+			   desc->db_trigger_cpu,
+			   desc->db_trigger_uk);
+		seq_printf(m, "\tProcess descriptor: 0x%x\n",
+			   desc->process_desc);
+		seq_printf(m, "\tWorkqueue address: 0x%x, size: 0x%x\n",
+			   desc->wq_addr, desc->wq_size);
+		seq_putc(m, '\n');
 
-		kunmap_atomic(log);
+		for_each_engine_masked(engine, dev_priv, client->engines, tmp) {
+			u32 guc_engine_id = engine->guc_id;
+			struct guc_execlist_context *lrc =
+						&desc->lrc[guc_engine_id];
+
+			seq_printf(m, "\t%s LRC:\n", engine->name);
+			seq_printf(m, "\t\tContext desc: 0x%x\n",
+				   lrc->context_desc);
+			seq_printf(m, "\t\tContext id: 0x%x\n", lrc->context_id);
+			seq_printf(m, "\t\tLRCA: 0x%x\n", lrc->ring_lrca);
+			seq_printf(m, "\t\tRing begin: 0x%x\n", lrc->ring_begin);
+			seq_printf(m, "\t\tRing end: 0x%x\n", lrc->ring_end);
+			seq_putc(m, '\n');
+		}
 	}
 
+	return 0;
+}
+
+static int i915_guc_log_dump(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = m->private;
+	struct drm_i915_private *dev_priv = node_to_i915(node);
+	bool dump_load_err = !!node->info_ent->data;
+	struct drm_i915_gem_object *obj = NULL;
+	u32 *log;
+	int i = 0;
+
+	if (dump_load_err)
+		obj = dev_priv->guc.load_err_log;
+	else if (dev_priv->guc.log.vma)
+		obj = dev_priv->guc.log.vma->obj;
+
+	if (!obj)
+		return 0;
+
+	log = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	if (IS_ERR(log)) {
+		DRM_DEBUG("Failed to pin object\n");
+		seq_puts(m, "(log data unaccessible)\n");
+		return PTR_ERR(log);
+	}
+
+	for (i = 0; i < obj->base.size / sizeof(u32); i += 4)
+		seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n",
+			   *(log + i), *(log + i + 1),
+			   *(log + i + 2), *(log + i + 3));
+
 	seq_putc(m, '\n');
 
+	i915_gem_object_unpin_map(obj);
+
 	return 0;
 }
 
 static int i915_guc_log_control_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
 	if (!dev_priv->guc.log.vma)
 		return -EINVAL;
@@ -2585,14 +2654,13 @@ static int i915_guc_log_control_get(void *data, u64 *val)
 
 static int i915_guc_log_control_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 	int ret;
 
 	if (!dev_priv->guc.log.vma)
 		return -EINVAL;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
 	if (ret)
 		return ret;
 
@@ -2600,7 +2668,7 @@ static int i915_guc_log_control_set(void *data, u64 val)
 	ret = i915_guc_log_control(dev_priv, val);
 	intel_runtime_pm_put(dev_priv);
 
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 	return ret;
 }
 
@@ -2859,7 +2927,8 @@ static int i915_dmc_info(struct seq_file *m, void *unused)
 	seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version),
 		   CSR_VERSION_MINOR(csr->version));
 
-	if (IS_SKYLAKE(dev_priv) && csr->version >= CSR_VERSION(1, 6)) {
+	if (IS_KABYLAKE(dev_priv) ||
+	    (IS_SKYLAKE(dev_priv) && csr->version >= CSR_VERSION(1, 6))) {
 		seq_printf(m, "DC3 -> DC5 count: %d\n",
 			   I915_READ(SKL_CSR_DC3_DC5_COUNT));
 		seq_printf(m, "DC5 -> DC6 count: %d\n",
@@ -3047,36 +3116,6 @@ static void intel_connector_info(struct seq_file *m,
 		intel_seq_print_mode(m, 2, mode);
 }
 
-static bool cursor_active(struct drm_i915_private *dev_priv, int pipe)
-{
-	u32 state;
-
-	if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
-		state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
-	else
-		state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
-
-	return state;
-}
-
-static bool cursor_position(struct drm_i915_private *dev_priv,
-			    int pipe, int *x, int *y)
-{
-	u32 pos;
-
-	pos = I915_READ(CURPOS(pipe));
-
-	*x = (pos >> CURSOR_X_SHIFT) & CURSOR_POS_MASK;
-	if (pos & (CURSOR_POS_SIGN << CURSOR_X_SHIFT))
-		*x = -*x;
-
-	*y = (pos >> CURSOR_Y_SHIFT) & CURSOR_POS_MASK;
-	if (pos & (CURSOR_POS_SIGN << CURSOR_Y_SHIFT))
-		*y = -*y;
-
-	return cursor_active(dev_priv, pipe);
-}
-
 static const char *plane_type(enum drm_plane_type type)
 {
 	switch (type) {
@@ -3099,17 +3138,17 @@ static const char *plane_rotation(unsigned int rotation)
 {
 	static char buf[48];
 	/*
-	 * According to doc only one DRM_ROTATE_ is allowed but this
+	 * According to doc only one DRM_MODE_ROTATE_ is allowed but this
 	 * will print them all to visualize if the values are misused
 	 */
 	snprintf(buf, sizeof(buf),
 		 "%s%s%s%s%s%s(0x%08x)",
-		 (rotation & DRM_ROTATE_0) ? "0 " : "",
-		 (rotation & DRM_ROTATE_90) ? "90 " : "",
-		 (rotation & DRM_ROTATE_180) ? "180 " : "",
-		 (rotation & DRM_ROTATE_270) ? "270 " : "",
-		 (rotation & DRM_REFLECT_X) ? "FLIPX " : "",
-		 (rotation & DRM_REFLECT_Y) ? "FLIPY " : "",
+		 (rotation & DRM_MODE_ROTATE_0) ? "0 " : "",
+		 (rotation & DRM_MODE_ROTATE_90) ? "90 " : "",
+		 (rotation & DRM_MODE_ROTATE_180) ? "180 " : "",
+		 (rotation & DRM_MODE_ROTATE_270) ? "270 " : "",
+		 (rotation & DRM_MODE_REFLECT_X) ? "FLIPX " : "",
+		 (rotation & DRM_MODE_REFLECT_Y) ? "FLIPY " : "",
 		 rotation);
 
 	return buf;
@@ -3198,9 +3237,7 @@ static int i915_display_info(struct seq_file *m, void *unused)
 	seq_printf(m, "CRTC info\n");
 	seq_printf(m, "---------\n");
 	for_each_intel_crtc(dev, crtc) {
-		bool active;
 		struct intel_crtc_state *pipe_config;
-		int x, y;
 
 		drm_modeset_lock(&crtc->base.mutex, NULL);
 		pipe_config = to_intel_crtc_state(crtc->base.state);
@@ -3212,14 +3249,18 @@ static int i915_display_info(struct seq_file *m, void *unused)
 			   yesno(pipe_config->dither), pipe_config->pipe_bpp);
 
 		if (pipe_config->base.active) {
+			struct intel_plane *cursor =
+				to_intel_plane(crtc->base.cursor);
+
 			intel_crtc_info(m, crtc);
 
-			active = cursor_position(dev_priv, crtc->pipe, &x, &y);
-			seq_printf(m, "\tcursor visible? %s, position (%d, %d), size %dx%d, addr 0x%08x, active? %s\n",
-				   yesno(crtc->cursor_base),
-				   x, y, crtc->base.cursor->state->crtc_w,
-				   crtc->base.cursor->state->crtc_h,
-				   crtc->cursor_addr, yesno(active));
+			seq_printf(m, "\tcursor visible? %s, position (%d, %d), size %dx%d, addr 0x%08x\n",
+				   yesno(cursor->base.state->visible),
+				   cursor->base.state->crtc_x,
+				   cursor->base.state->crtc_y,
+				   cursor->base.state->crtc_w,
+				   cursor->base.state->crtc_h,
+				   cursor->cursor.base);
 			intel_scaler_info(m, crtc);
 			intel_plane_info(m, crtc);
 		}
@@ -3320,7 +3361,7 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 
 		if (i915.enable_execlists) {
 			u32 ptr, read, write;
-			struct rb_node *rb;
+			unsigned int idx;
 
 			seq_printf(m, "\tExeclist status: 0x%08x %08x\n",
 				   I915_READ(RING_EXECLIST_STATUS_LO(engine)),
@@ -3338,8 +3379,7 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 			if (read > write)
 				write += GEN8_CSB_ENTRIES;
 			while (read < write) {
-				unsigned int idx = ++read % GEN8_CSB_ENTRIES;
-
+				idx = ++read % GEN8_CSB_ENTRIES;
 				seq_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
 					   idx,
 					   I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
@@ -3347,28 +3387,30 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 			}
 
 			rcu_read_lock();
-			rq = READ_ONCE(engine->execlist_port[0].request);
-			if (rq) {
-				seq_printf(m, "\t\tELSP[0] count=%d, ",
-					   engine->execlist_port[0].count);
-				print_request(m, rq, "rq: ");
-			} else {
-				seq_printf(m, "\t\tELSP[0] idle\n");
-			}
-			rq = READ_ONCE(engine->execlist_port[1].request);
-			if (rq) {
-				seq_printf(m, "\t\tELSP[1] count=%d, ",
-					   engine->execlist_port[1].count);
-				print_request(m, rq, "rq: ");
-			} else {
-				seq_printf(m, "\t\tELSP[1] idle\n");
+			for (idx = 0; idx < ARRAY_SIZE(engine->execlist_port); idx++) {
+				unsigned int count;
+
+				rq = port_unpack(&engine->execlist_port[idx],
+						 &count);
+				if (rq) {
+					seq_printf(m, "\t\tELSP[%d] count=%d, ",
+						   idx, count);
+					print_request(m, rq, "rq: ");
+				} else {
+					seq_printf(m, "\t\tELSP[%d] idle\n",
+						   idx);
+				}
 			}
 			rcu_read_unlock();
 
 			spin_lock_irq(&engine->timeline->lock);
-			for (rb = engine->execlist_first; rb; rb = rb_next(rb)) {
-				rq = rb_entry(rb, typeof(*rq), priotree.node);
-				print_request(m, rq, "\t\tQ ");
+			for (rb = engine->execlist_first; rb; rb = rb_next(rb)){
+				struct i915_priolist *p =
+					rb_entry(rb, typeof(*p), node);
+
+				list_for_each_entry(rq, &p->requests,
+						    priotree.link)
+					print_request(m, rq, "\t\tQ ");
 			}
 			spin_unlock_irq(&engine->timeline->lock);
 		} else if (INTEL_GEN(dev_priv) > 6) {
@@ -3708,16 +3750,10 @@ static ssize_t i915_displayport_test_active_write(struct file *file,
 	if (len == 0)
 		return 0;
 
-	input_buffer = kmalloc(len + 1, GFP_KERNEL);
-	if (!input_buffer)
-		return -ENOMEM;
-
-	if (copy_from_user(input_buffer, ubuf, len)) {
-		status = -EFAULT;
-		goto out;
-	}
+	input_buffer = memdup_user_nul(ubuf, len);
+	if (IS_ERR(input_buffer))
+		return PTR_ERR(input_buffer);
 
-	input_buffer[len] = '\0';
 	DRM_DEBUG_DRIVER("Copied %d bytes from user\n", (unsigned int)len);
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
@@ -3743,7 +3779,6 @@ static ssize_t i915_displayport_test_active_write(struct file *file,
 		}
 	}
 	drm_connector_list_iter_end(&conn_iter);
-out:
 	kfree(input_buffer);
 	if (status < 0)
 		return status;
@@ -3904,6 +3939,8 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 		num_levels = 3;
 	else if (IS_VALLEYVIEW(dev_priv))
 		num_levels = 1;
+	else if (IS_G4X(dev_priv))
+		num_levels = 3;
 	else
 		num_levels = ilk_wm_max_level(dev_priv) + 1;
 
@@ -3916,8 +3953,10 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 		 * - WM1+ latency values in 0.5us units
 		 * - latencies are in us on gen9/vlv/chv
 		 */
-		if (INTEL_GEN(dev_priv) >= 9 || IS_VALLEYVIEW(dev_priv) ||
-		    IS_CHERRYVIEW(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 9 ||
+		    IS_VALLEYVIEW(dev_priv) ||
+		    IS_CHERRYVIEW(dev_priv) ||
+		    IS_G4X(dev_priv))
 			latency *= 10;
 		else if (level > 0)
 			latency *= 5;
@@ -3978,7 +4017,7 @@ static int pri_wm_latency_open(struct inode *inode, struct file *file)
 {
 	struct drm_i915_private *dev_priv = inode->i_private;
 
-	if (INTEL_GEN(dev_priv) < 5)
+	if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
 		return -ENODEV;
 
 	return single_open(file, pri_wm_latency_show, dev_priv);
@@ -4020,6 +4059,8 @@ static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
 		num_levels = 3;
 	else if (IS_VALLEYVIEW(dev_priv))
 		num_levels = 1;
+	else if (IS_G4X(dev_priv))
+		num_levels = 3;
 	else
 		num_levels = ilk_wm_max_level(dev_priv) + 1;
 
@@ -4268,26 +4309,27 @@ i915_drop_caches_set(void *data, u64 val)
 {
 	struct drm_i915_private *dev_priv = data;
 	struct drm_device *dev = &dev_priv->drm;
-	int ret;
+	int ret = 0;
 
 	DRM_DEBUG("Dropping caches: 0x%08llx\n", val);
 
 	/* No need to check and wait for gpu resets, only libdrm auto-restarts
 	 * on ioctls on -EAGAIN. */
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	if (val & DROP_ACTIVE) {
-		ret = i915_gem_wait_for_idle(dev_priv,
-					     I915_WAIT_INTERRUPTIBLE |
-					     I915_WAIT_LOCKED);
+	if (val & (DROP_ACTIVE | DROP_RETIRE)) {
+		ret = mutex_lock_interruptible(&dev->struct_mutex);
 		if (ret)
-			goto unlock;
-	}
+			return ret;
+
+		if (val & DROP_ACTIVE)
+			ret = i915_gem_wait_for_idle(dev_priv,
+						     I915_WAIT_INTERRUPTIBLE |
+						     I915_WAIT_LOCKED);
 
-	if (val & DROP_RETIRE)
-		i915_gem_retire_requests(dev_priv);
+		if (val & DROP_RETIRE)
+			i915_gem_retire_requests(dev_priv);
+
+		mutex_unlock(&dev->struct_mutex);
+	}
 
 	lockdep_set_current_reclaim_state(GFP_KERNEL);
 	if (val & DROP_BOUND)
@@ -4300,9 +4342,6 @@ i915_drop_caches_set(void *data, u64 val)
 		i915_gem_shrink_all(dev_priv);
 	lockdep_clear_current_reclaim_state();
 
-unlock:
-	mutex_unlock(&dev->struct_mutex);
-
 	if (val & DROP_FREED) {
 		synchronize_rcu();
 		i915_gem_drain_freed_objects(dev_priv);
@@ -4780,6 +4819,8 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_guc_info", i915_guc_info, 0},
 	{"i915_guc_load_status", i915_guc_load_status_info, 0},
 	{"i915_guc_log_dump", i915_guc_log_dump, 0},
+	{"i915_guc_load_err_log_dump", i915_guc_log_dump, 0, (void *)1},
+	{"i915_guc_stage_pool", i915_guc_stage_pool, 0},
 	{"i915_huc_load_status", i915_huc_load_status_info, 0},
 	{"i915_frequency_info", i915_frequency_info, 0},
 	{"i915_hangcheck_info", i915_hangcheck_info, 0},
@@ -4838,7 +4879,7 @@ static const struct i915_debugfs_files {
 	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
 	{"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
 	{"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
-	{"i915_fbc_false_color", &i915_fbc_fc_fops},
+	{"i915_fbc_false_color", &i915_fbc_false_color_fops},
 	{"i915_dp_test_data", &i915_displayport_test_data_fops},
 	{"i915_dp_test_type", &i915_displayport_test_type_fops},
 	{"i915_dp_test_active", &i915_displayport_test_active_fops},
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 48428672fc6e..ee2325b180e7 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -139,6 +139,9 @@ static enum intel_pch intel_virt_detect_pch(struct drm_i915_private *dev_priv)
 	} else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		ret = PCH_SPT;
 		DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
+	} else if (IS_COFFEELAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+		ret = PCH_CNP;
+		DRM_DEBUG_KMS("Assuming CannonPoint PCH\n");
 	}
 
 	return ret;
@@ -170,24 +173,29 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv)
 	while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
 		if (pch->vendor == PCI_VENDOR_ID_INTEL) {
 			unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
-			dev_priv->pch_id = id;
+			unsigned short id_ext = pch->device &
+				INTEL_PCH_DEVICE_ID_MASK_EXT;
 
 			if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_IBX;
 				DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
 				WARN_ON(!IS_GEN5(dev_priv));
 			} else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found CougarPoint PCH\n");
 				WARN_ON(!(IS_GEN6(dev_priv) ||
 					IS_IVYBRIDGE(dev_priv)));
 			} else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
 				/* PantherPoint is CPT compatible */
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found PantherPoint PCH\n");
 				WARN_ON(!(IS_GEN6(dev_priv) ||
 					IS_IVYBRIDGE(dev_priv)));
 			} else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_LPT;
 				DRM_DEBUG_KMS("Found LynxPoint PCH\n");
 				WARN_ON(!IS_HASWELL(dev_priv) &&
@@ -195,6 +203,7 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv)
 				WARN_ON(IS_HSW_ULT(dev_priv) ||
 					IS_BDW_ULT(dev_priv));
 			} else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_LPT;
 				DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
 				WARN_ON(!IS_HASWELL(dev_priv) &&
@@ -202,20 +211,35 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv)
 				WARN_ON(!IS_HSW_ULT(dev_priv) &&
 					!IS_BDW_ULT(dev_priv));
 			} else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
 				WARN_ON(!IS_SKYLAKE(dev_priv) &&
 					!IS_KABYLAKE(dev_priv));
-			} else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
+			} else if (id_ext == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id_ext;
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
 				WARN_ON(!IS_SKYLAKE(dev_priv) &&
 					!IS_KABYLAKE(dev_priv));
 			} else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_KBP;
 				DRM_DEBUG_KMS("Found KabyPoint PCH\n");
 				WARN_ON(!IS_SKYLAKE(dev_priv) &&
 					!IS_KABYLAKE(dev_priv));
+			} else if (id == INTEL_PCH_CNP_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id;
+				dev_priv->pch_type = PCH_CNP;
+				DRM_DEBUG_KMS("Found CannonPoint PCH\n");
+				WARN_ON(!IS_CANNONLAKE(dev_priv) &&
+					!IS_COFFEELAKE(dev_priv));
+			} else if (id_ext == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE) {
+				dev_priv->pch_id = id_ext;
+				dev_priv->pch_type = PCH_CNP;
+				DRM_DEBUG_KMS("Found CannonPoint LP PCH\n");
+				WARN_ON(!IS_CANNONLAKE(dev_priv) &&
+					!IS_COFFEELAKE(dev_priv));
 			} else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
 				   (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
 				   ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
@@ -223,6 +247,7 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv)
 					    PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
 				    pch->subsystem_device ==
 					    PCI_SUBDEVICE_ID_QEMU)) {
+				dev_priv->pch_id = id;
 				dev_priv->pch_type =
 					intel_virt_detect_pch(dev_priv);
 			} else
@@ -350,6 +375,8 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_EXEC_SOFTPIN:
 	case I915_PARAM_HAS_EXEC_ASYNC:
 	case I915_PARAM_HAS_EXEC_FENCE:
+	case I915_PARAM_HAS_EXEC_CAPTURE:
+	case I915_PARAM_HAS_EXEC_BATCH_FIRST:
 		/* For the time being all of these are always true;
 		 * if some supported hardware does not have one of these
 		 * features this value needs to be provided from
@@ -357,6 +384,16 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		 */
 		value = 1;
 		break;
+	case I915_PARAM_SLICE_MASK:
+		value = INTEL_INFO(dev_priv)->sseu.slice_mask;
+		if (!value)
+			return -ENODEV;
+		break;
+	case I915_PARAM_SUBSLICE_MASK:
+		value = INTEL_INFO(dev_priv)->sseu.subslice_mask;
+		if (!value)
+			return -ENODEV;
+		break;
 	default:
 		DRM_DEBUG("Unknown parameter %d\n", param->param);
 		return -EINVAL;
@@ -552,6 +589,7 @@ static void i915_gem_fini(struct drm_i915_private *dev_priv)
 	intel_uc_fini_hw(dev_priv);
 	i915_gem_cleanup_engines(dev_priv);
 	i915_gem_context_fini(dev_priv);
+	i915_gem_cleanup_userptr(dev_priv);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	i915_gem_drain_freed_objects(dev_priv);
@@ -834,10 +872,6 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	intel_uc_init_early(dev_priv);
 	i915_memcpy_init_early(dev_priv);
 
-	ret = intel_engines_init_early(dev_priv);
-	if (ret)
-		return ret;
-
 	ret = i915_workqueues_init(dev_priv);
 	if (ret < 0)
 		goto err_engines;
@@ -855,7 +889,7 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	intel_init_audio_hooks(dev_priv);
 	ret = i915_gem_load_init(dev_priv);
 	if (ret < 0)
-		goto err_workqueues;
+		goto err_irq;
 
 	intel_display_crc_init(dev_priv);
 
@@ -867,7 +901,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 
 	return 0;
 
-err_workqueues:
+err_irq:
+	intel_irq_fini(dev_priv);
 	i915_workqueues_cleanup(dev_priv);
 err_engines:
 	i915_engines_cleanup(dev_priv);
@@ -882,6 +917,7 @@ static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
 {
 	i915_perf_fini(dev_priv);
 	i915_gem_load_cleanup(dev_priv);
+	intel_irq_fini(dev_priv);
 	i915_workqueues_cleanup(dev_priv);
 	i915_engines_cleanup(dev_priv);
 }
@@ -947,14 +983,21 @@ static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
 
 	ret = i915_mmio_setup(dev_priv);
 	if (ret < 0)
-		goto put_bridge;
+		goto err_bridge;
 
 	intel_uncore_init(dev_priv);
+
+	ret = intel_engines_init_mmio(dev_priv);
+	if (ret)
+		goto err_uncore;
+
 	i915_gem_init_mmio(dev_priv);
 
 	return 0;
 
-put_bridge:
+err_uncore:
+	intel_uncore_fini(dev_priv);
+err_bridge:
 	pci_dev_put(dev_priv->bridge_dev);
 
 	return ret;
@@ -991,6 +1034,8 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 	DRM_DEBUG_DRIVER("use GPU semaphores? %s\n", yesno(i915.semaphores));
 
 	intel_uc_sanitize_options(dev_priv);
+
+	intel_gvt_sanitize_options(dev_priv);
 }
 
 /**
@@ -1213,9 +1258,8 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct drm_i915_private *dev_priv;
 	int ret;
 
-	/* Enable nuclear pageflip on ILK+, except vlv/chv */
-	if (!i915.nuclear_pageflip &&
-	    (match_info->gen < 5 || match_info->has_gmch_display))
+	/* Enable nuclear pageflip on ILK+ */
+	if (!i915.nuclear_pageflip && match_info->gen < 5)
 		driver.driver_features &= ~DRIVER_ATOMIC;
 
 	ret = -ENOMEM;
@@ -2454,9 +2498,6 @@ static int intel_runtime_resume(struct device *kdev)
 
 	intel_guc_resume(dev_priv);
 
-	if (IS_GEN6(dev_priv))
-		intel_init_pch_refclk(dev_priv);
-
 	if (IS_GEN9_LP(dev_priv)) {
 		bxt_disable_dc9(dev_priv);
 		bxt_display_core_init(dev_priv, true);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 2c453a4e97d5..e1f7c97a338a 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -37,7 +37,7 @@
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <linux/backlight.h>
-#include <linux/hashtable.h>
+#include <linux/hash.h>
 #include <linux/intel-iommu.h>
 #include <linux/kref.h>
 #include <linux/pm_qos.h>
@@ -55,6 +55,7 @@
 #include "i915_reg.h"
 #include "i915_utils.h"
 
+#include "intel_uncore.h"
 #include "intel_bios.h"
 #include "intel_dpll_mgr.h"
 #include "intel_uc.h"
@@ -79,8 +80,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20170403"
-#define DRIVER_TIMESTAMP	1491198738
+#define DRIVER_DATE		"20170619"
+#define DRIVER_TIMESTAMP	1497857498
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -114,6 +115,13 @@ typedef struct {
 	fp; \
 })
 
+static inline bool is_fixed16_zero(uint_fixed_16_16_t val)
+{
+	if (val.val == 0)
+		return true;
+	return false;
+}
+
 static inline uint_fixed_16_16_t u32_to_fixed_16_16(uint32_t val)
 {
 	uint_fixed_16_16_t fp;
@@ -152,8 +160,39 @@ static inline uint_fixed_16_16_t max_fixed_16_16(uint_fixed_16_16_t max1,
 	return max;
 }
 
-static inline uint_fixed_16_16_t fixed_16_16_div_round_up(uint32_t val,
-							  uint32_t d)
+static inline uint32_t div_round_up_fixed16(uint_fixed_16_16_t val,
+					    uint_fixed_16_16_t d)
+{
+	return DIV_ROUND_UP(val.val, d.val);
+}
+
+static inline uint32_t mul_round_up_u32_fixed16(uint32_t val,
+						uint_fixed_16_16_t mul)
+{
+	uint64_t intermediate_val;
+	uint32_t result;
+
+	intermediate_val = (uint64_t) val * mul.val;
+	intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16);
+	WARN_ON(intermediate_val >> 32);
+	result = clamp_t(uint32_t, intermediate_val, 0, ~0);
+	return result;
+}
+
+static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val,
+					     uint_fixed_16_16_t mul)
+{
+	uint64_t intermediate_val;
+	uint_fixed_16_16_t fp;
+
+	intermediate_val = (uint64_t) val.val * mul.val;
+	intermediate_val = intermediate_val >> 16;
+	WARN_ON(intermediate_val >> 32);
+	fp.val = clamp_t(uint32_t, intermediate_val, 0, ~0);
+	return fp;
+}
+
+static inline uint_fixed_16_16_t fixed_16_16_div(uint32_t val, uint32_t d)
 {
 	uint_fixed_16_16_t fp, res;
 
@@ -162,8 +201,7 @@ static inline uint_fixed_16_16_t fixed_16_16_div_round_up(uint32_t val,
 	return res;
 }
 
-static inline uint_fixed_16_16_t fixed_16_16_div_round_up_u64(uint32_t val,
-							      uint32_t d)
+static inline uint_fixed_16_16_t fixed_16_16_div_u64(uint32_t val, uint32_t d)
 {
 	uint_fixed_16_16_t res;
 	uint64_t interm_val;
@@ -176,6 +214,17 @@ static inline uint_fixed_16_16_t fixed_16_16_div_round_up_u64(uint32_t val,
 	return res;
 }
 
+static inline uint32_t div_round_up_u32_fixed16(uint32_t val,
+						uint_fixed_16_16_t d)
+{
+	uint64_t interm_val;
+
+	interm_val = (uint64_t)val << 16;
+	interm_val = DIV_ROUND_UP_ULL(interm_val, d.val);
+	WARN_ON(interm_val >> 32);
+	return clamp_t(uint32_t, interm_val, 0, ~0);
+}
+
 static inline uint_fixed_16_16_t mul_u32_fixed_16_16(uint32_t val,
 						     uint_fixed_16_16_t mul)
 {
@@ -677,116 +726,6 @@ struct drm_i915_display_funcs {
 	void (*load_luts)(struct drm_crtc_state *crtc_state);
 };
 
-enum forcewake_domain_id {
-	FW_DOMAIN_ID_RENDER = 0,
-	FW_DOMAIN_ID_BLITTER,
-	FW_DOMAIN_ID_MEDIA,
-
-	FW_DOMAIN_ID_COUNT
-};
-
-enum forcewake_domains {
-	FORCEWAKE_RENDER = BIT(FW_DOMAIN_ID_RENDER),
-	FORCEWAKE_BLITTER = BIT(FW_DOMAIN_ID_BLITTER),
-	FORCEWAKE_MEDIA	= BIT(FW_DOMAIN_ID_MEDIA),
-	FORCEWAKE_ALL = (FORCEWAKE_RENDER |
-			 FORCEWAKE_BLITTER |
-			 FORCEWAKE_MEDIA)
-};
-
-#define FW_REG_READ  (1)
-#define FW_REG_WRITE (2)
-
-enum decoupled_power_domain {
-	GEN9_DECOUPLED_PD_BLITTER = 0,
-	GEN9_DECOUPLED_PD_RENDER,
-	GEN9_DECOUPLED_PD_MEDIA,
-	GEN9_DECOUPLED_PD_ALL
-};
-
-enum decoupled_ops {
-	GEN9_DECOUPLED_OP_WRITE = 0,
-	GEN9_DECOUPLED_OP_READ
-};
-
-enum forcewake_domains
-intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
-			       i915_reg_t reg, unsigned int op);
-
-struct intel_uncore_funcs {
-	void (*force_wake_get)(struct drm_i915_private *dev_priv,
-			       enum forcewake_domains domains);
-	void (*force_wake_put)(struct drm_i915_private *dev_priv,
-			       enum forcewake_domains domains);
-
-	uint8_t  (*mmio_readb)(struct drm_i915_private *dev_priv,
-			       i915_reg_t r, bool trace);
-	uint16_t (*mmio_readw)(struct drm_i915_private *dev_priv,
-			       i915_reg_t r, bool trace);
-	uint32_t (*mmio_readl)(struct drm_i915_private *dev_priv,
-			       i915_reg_t r, bool trace);
-	uint64_t (*mmio_readq)(struct drm_i915_private *dev_priv,
-			       i915_reg_t r, bool trace);
-
-	void (*mmio_writeb)(struct drm_i915_private *dev_priv,
-			    i915_reg_t r, uint8_t val, bool trace);
-	void (*mmio_writew)(struct drm_i915_private *dev_priv,
-			    i915_reg_t r, uint16_t val, bool trace);
-	void (*mmio_writel)(struct drm_i915_private *dev_priv,
-			    i915_reg_t r, uint32_t val, bool trace);
-};
-
-struct intel_forcewake_range {
-	u32 start;
-	u32 end;
-
-	enum forcewake_domains domains;
-};
-
-struct intel_uncore {
-	spinlock_t lock; /** lock is also taken in irq contexts. */
-
-	const struct intel_forcewake_range *fw_domains_table;
-	unsigned int fw_domains_table_entries;
-
-	struct notifier_block pmic_bus_access_nb;
-	struct intel_uncore_funcs funcs;
-
-	unsigned fifo_count;
-
-	enum forcewake_domains fw_domains;
-	enum forcewake_domains fw_domains_active;
-
-	u32 fw_set;
-	u32 fw_clear;
-	u32 fw_reset;
-
-	struct intel_uncore_forcewake_domain {
-		enum forcewake_domain_id id;
-		enum forcewake_domains mask;
-		unsigned wake_count;
-		struct hrtimer timer;
-		i915_reg_t reg_set;
-		i915_reg_t reg_ack;
-	} fw_domain[FW_DOMAIN_ID_COUNT];
-
-	int unclaimed_mmio_check;
-};
-
-#define __mask_next_bit(mask) ({					\
-	int __idx = ffs(mask) - 1;					\
-	mask &= ~BIT(__idx);						\
-	__idx;								\
-})
-
-/* Iterate over initialised fw domains */
-#define for_each_fw_domain_masked(domain__, mask__, dev_priv__, tmp__) \
-	for (tmp__ = (mask__); \
-	     tmp__ ? (domain__ = &(dev_priv__)->uncore.fw_domain[__mask_next_bit(tmp__)]), 1 : 0;)
-
-#define for_each_fw_domain(domain__, dev_priv__, tmp__) \
-	for_each_fw_domain_masked(domain__, (dev_priv__)->uncore.fw_domains, dev_priv__, tmp__)
-
 #define CSR_VERSION(major, minor)	((major) << 16 | (minor))
 #define CSR_VERSION_MAJOR(version)	((version) >> 16)
 #define CSR_VERSION_MINOR(version)	((version) & 0xffff)
@@ -813,7 +752,6 @@ struct intel_csr {
 	func(has_aliasing_ppgtt); \
 	func(has_csr); \
 	func(has_ddi); \
-	func(has_decoupled_mmio); \
 	func(has_dp_mst); \
 	func(has_fbc); \
 	func(has_fpga_dbg); \
@@ -822,8 +760,8 @@ struct intel_csr {
 	func(has_gmbus_irq); \
 	func(has_gmch_display); \
 	func(has_guc); \
+	func(has_guc_ct); \
 	func(has_hotplug); \
-	func(has_hw_contexts); \
 	func(has_l3_dpf); \
 	func(has_llc); \
 	func(has_logical_ring_contexts); \
@@ -888,6 +826,8 @@ enum intel_platform {
 	INTEL_BROXTON,
 	INTEL_KABYLAKE,
 	INTEL_GEMINILAKE,
+	INTEL_COFFEELAKE,
+	INTEL_CANNONLAKE,
 	INTEL_MAX_PLATFORMS
 };
 
@@ -1026,6 +966,9 @@ struct i915_gpu_state {
 			u32 *pages[0];
 		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
 
+		struct drm_i915_error_object **user_bo;
+		long user_bo_count;
+
 		struct drm_i915_error_object *wa_ctx;
 
 		struct drm_i915_error_request {
@@ -1210,6 +1153,7 @@ enum intel_pch {
 	PCH_LPT,	/* Lynxpoint PCH */
 	PCH_SPT,        /* Sunrisepoint PCH */
 	PCH_KBP,        /* Kabypoint PCH */
+	PCH_CNP,        /* Cannonpoint PCH */
 	PCH_NOP,
 };
 
@@ -1218,11 +1162,9 @@ enum intel_sbi_destination {
 	SBI_MPHY,
 };
 
-#define QUIRK_PIPEA_FORCE (1<<0)
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
 #define QUIRK_INVERT_BRIGHTNESS (1<<2)
 #define QUIRK_BACKLIGHT_PRESENT (1<<3)
-#define QUIRK_PIPEB_FORCE (1<<4)
 #define QUIRK_PIN_SWIZZLED_PAGES (1<<5)
 
 struct intel_fbdev;
@@ -1513,10 +1455,13 @@ struct i915_gem_mm {
 	struct list_head fence_list;
 
 	/**
-	 * Are we in a non-interruptible section of code like
-	 * modesetting?
+	 * Workqueue to fault in userptr pages, flushed by the execbuf
+	 * when required but otherwise left to userspace to try again
+	 * on EAGAIN.
 	 */
-	bool interruptible;
+	struct workqueue_struct *userptr_wq;
+
+	u64 unordered_timeline;
 
 	/* the indicator for dispatch video commands on two BSD rings */
 	atomic_t bsd_engine_dispatch_index;
@@ -1567,7 +1512,7 @@ struct i915_gpu_error {
 	 *
 	 * This is a counter which gets incremented when reset is triggered,
 	 *
-	 * Before the reset commences, the I915_RESET_IN_PROGRESS bit is set
+	 * Before the reset commences, the I915_RESET_BACKOFF bit is set
 	 * meaning that any waiters holding onto the struct_mutex should
 	 * relinquish the lock immediately in order for the reset to start.
 	 *
@@ -1764,13 +1709,15 @@ struct ilk_wm_values {
 	enum intel_ddb_partitioning partitioning;
 };
 
-struct vlv_pipe_wm {
+struct g4x_pipe_wm {
 	uint16_t plane[I915_MAX_PLANES];
+	uint16_t fbc;
 };
 
-struct vlv_sr_wm {
+struct g4x_sr_wm {
 	uint16_t plane;
 	uint16_t cursor;
+	uint16_t fbc;
 };
 
 struct vlv_wm_ddl_values {
@@ -1778,13 +1725,22 @@ struct vlv_wm_ddl_values {
 };
 
 struct vlv_wm_values {
-	struct vlv_pipe_wm pipe[3];
-	struct vlv_sr_wm sr;
+	struct g4x_pipe_wm pipe[3];
+	struct g4x_sr_wm sr;
 	struct vlv_wm_ddl_values ddl[3];
 	uint8_t level;
 	bool cxsr;
 };
 
+struct g4x_wm_values {
+	struct g4x_pipe_wm pipe[2];
+	struct g4x_sr_wm sr;
+	struct g4x_sr_wm hpll;
+	bool cxsr;
+	bool hpll_en;
+	bool fbc_en;
+};
+
 struct skl_ddb_entry {
 	uint16_t start, end;	/* in number of blocks, 'end' is exclusive */
 };
@@ -2068,9 +2024,17 @@ struct i915_oa_ops {
 	void (*init_oa_buffer)(struct drm_i915_private *dev_priv);
 
 	/**
-	 * @enable_metric_set: Applies any MUX configuration to set up the
-	 * Boolean and Custom (B/C) counters that are part of the counter
-	 * reports being sampled. May apply system constraints such as
+	 * @select_metric_set: The auto generated code that checks whether a
+	 * requested OA config is applicable to the system and if so sets up
+	 * the mux, oa and flex eu register config pointers according to the
+	 * current dev_priv->perf.oa.metrics_set.
+	 */
+	int (*select_metric_set)(struct drm_i915_private *dev_priv);
+
+	/**
+	 * @enable_metric_set: Selects and applies any MUX configuration to set
+	 * up the Boolean and Custom (B/C) counters that are part of the
+	 * counter reports being sampled. May apply system constraints such as
 	 * disabling EU clock gating as required.
 	 */
 	int (*enable_metric_set)(struct drm_i915_private *dev_priv);
@@ -2101,20 +2065,13 @@ struct i915_oa_ops {
 		    size_t *offset);
 
 	/**
-	 * @oa_buffer_is_empty: Check if OA buffer empty (false positives OK)
-	 *
-	 * This is either called via fops or the poll check hrtimer (atomic
-	 * ctx) without any locks taken.
+	 * @oa_hw_tail_read: read the OA tail pointer register
 	 *
-	 * It's safe to read OA config state here unlocked, assuming that this
-	 * is only called while the stream is enabled, while the global OA
-	 * configuration can't be modified.
-	 *
-	 * Efficiency is more important than avoiding some false positives
-	 * here, which will be handled gracefully - likely resulting in an
-	 * %EAGAIN error for userspace.
+	 * In particular this enables us to share all the fiddly code for
+	 * handling the OA unit tail pointer race that affects multiple
+	 * generations.
 	 */
-	bool (*oa_buffer_is_empty)(struct drm_i915_private *dev_priv);
+	u32 (*oa_hw_tail_read)(struct drm_i915_private *dev_priv);
 };
 
 struct intel_cdclk_state {
@@ -2128,6 +2085,7 @@ struct drm_i915_private {
 	struct kmem_cache *vmas;
 	struct kmem_cache *requests;
 	struct kmem_cache *dependencies;
+	struct kmem_cache *priorities;
 
 	const struct intel_device_info info;
 
@@ -2363,7 +2321,6 @@ struct drm_i915_private {
 	 */
 	struct mutex av_mutex;
 
-	uint32_t hw_context_size;
 	struct list_head context_list;
 
 	u32 fdi_rx_config;
@@ -2414,6 +2371,7 @@ struct drm_i915_private {
 			struct ilk_wm_values hw;
 			struct skl_wm_values skl_hw;
 			struct vlv_wm_values vlv;
+			struct g4x_wm_values g4x;
 		};
 
 		uint8_t max_level;
@@ -2444,8 +2402,6 @@ struct drm_i915_private {
 		struct mutex lock;
 		struct list_head streams;
 
-		spinlock_t hook_lock;
-
 		struct {
 			struct i915_perf_stream *exclusive_stream;
 
@@ -2455,27 +2411,109 @@ struct drm_i915_private {
 			wait_queue_head_t poll_wq;
 			bool pollin;
 
+			/**
+			 * For rate limiting any notifications of spurious
+			 * invalid OA reports
+			 */
+			struct ratelimit_state spurious_report_rs;
+
 			bool periodic;
 			int period_exponent;
 			int timestamp_frequency;
 
-			int tail_margin;
-
 			int metrics_set;
 
-			const struct i915_oa_reg *mux_regs;
-			int mux_regs_len;
+			const struct i915_oa_reg *mux_regs[6];
+			int mux_regs_lens[6];
+			int n_mux_configs;
+
 			const struct i915_oa_reg *b_counter_regs;
 			int b_counter_regs_len;
+			const struct i915_oa_reg *flex_regs;
+			int flex_regs_len;
 
 			struct {
 				struct i915_vma *vma;
 				u8 *vaddr;
+				u32 last_ctx_id;
 				int format;
 				int format_size;
+
+				/**
+				 * Locks reads and writes to all head/tail state
+				 *
+				 * Consider: the head and tail pointer state
+				 * needs to be read consistently from a hrtimer
+				 * callback (atomic context) and read() fop
+				 * (user context) with tail pointer updates
+				 * happening in atomic context and head updates
+				 * in user context and the (unlikely)
+				 * possibility of read() errors needing to
+				 * reset all head/tail state.
+				 *
+				 * Note: Contention or performance aren't
+				 * currently a significant concern here
+				 * considering the relatively low frequency of
+				 * hrtimer callbacks (5ms period) and that
+				 * reads typically only happen in response to a
+				 * hrtimer event and likely complete before the
+				 * next callback.
+				 *
+				 * Note: This lock is not held *while* reading
+				 * and copying data to userspace so the value
+				 * of head observed in htrimer callbacks won't
+				 * represent any partial consumption of data.
+				 */
+				spinlock_t ptr_lock;
+
+				/**
+				 * One 'aging' tail pointer and one 'aged'
+				 * tail pointer ready to used for reading.
+				 *
+				 * Initial values of 0xffffffff are invalid
+				 * and imply that an update is required
+				 * (and should be ignored by an attempted
+				 * read)
+				 */
+				struct {
+					u32 offset;
+				} tails[2];
+
+				/**
+				 * Index for the aged tail ready to read()
+				 * data up to.
+				 */
+				unsigned int aged_tail_idx;
+
+				/**
+				 * A monotonic timestamp for when the current
+				 * aging tail pointer was read; used to
+				 * determine when it is old enough to trust.
+				 */
+				u64 aging_timestamp;
+
+				/**
+				 * Although we can always read back the head
+				 * pointer register, we prefer to avoid
+				 * trusting the HW state, just to avoid any
+				 * risk that some hardware condition could
+				 * somehow bump the head pointer unpredictably
+				 * and cause us to forward the wrong OA buffer
+				 * data to userspace.
+				 */
+				u32 head;
 			} oa_buffer;
 
 			u32 gen7_latched_oastatus1;
+			u32 ctx_oactxctrl_offset;
+			u32 ctx_flexeu0_offset;
+
+			/**
+			 * The RPT_ID/reason field for Gen8+ includes a bit
+			 * to determine if the CTX ID in the report is valid
+			 * but the specific bit differs between Gen 8 and 9
+			 */
+			u32 gen8_valid_ctx_bit;
 
 			struct i915_oa_ops ops;
 			const struct i915_oa_format *oa_formats;
@@ -2751,6 +2789,8 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define IS_BROXTON(dev_priv)	((dev_priv)->info.platform == INTEL_BROXTON)
 #define IS_KABYLAKE(dev_priv)	((dev_priv)->info.platform == INTEL_KABYLAKE)
 #define IS_GEMINILAKE(dev_priv)	((dev_priv)->info.platform == INTEL_GEMINILAKE)
+#define IS_COFFEELAKE(dev_priv)	((dev_priv)->info.platform == INTEL_COFFEELAKE)
+#define IS_CANNONLAKE(dev_priv)	((dev_priv)->info.platform == INTEL_CANNONLAKE)
 #define IS_MOBILE(dev_priv)	((dev_priv)->info.is_mobile)
 #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
 				    (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
@@ -2786,10 +2826,18 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define IS_KBL_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x590E || \
 				 INTEL_DEVID(dev_priv) == 0x5915 || \
 				 INTEL_DEVID(dev_priv) == 0x591E)
+#define IS_SKL_GT2(dev_priv)	(IS_SKYLAKE(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0010)
 #define IS_SKL_GT3(dev_priv)	(IS_SKYLAKE(dev_priv) && \
 				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
 #define IS_SKL_GT4(dev_priv)	(IS_SKYLAKE(dev_priv) && \
 				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030)
+#define IS_KBL_GT2(dev_priv)	(IS_KABYLAKE(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0010)
+#define IS_KBL_GT3(dev_priv)	(IS_KABYLAKE(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
+#define IS_CFL_ULT(dev_priv)	(IS_COFFEELAKE(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0)
 
 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
 
@@ -2828,6 +2876,12 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define IS_GLK_REVID(dev_priv, since, until) \
 	(IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until))
 
+#define CNL_REVID_A0		0x0
+#define CNL_REVID_B0		0x1
+
+#define IS_CNL_REVID(p, since, until) \
+	(IS_CANNONLAKE(p) && IS_REVID(p, since, until))
+
 /*
  * The genX designation typically refers to the render engine, so render
  * capability related checks should use IS_GEN, while display and other checks
@@ -2842,6 +2896,7 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define IS_GEN7(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(6)))
 #define IS_GEN8(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(7)))
 #define IS_GEN9(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(8)))
+#define IS_GEN10(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(9)))
 
 #define IS_LP(dev_priv)	(INTEL_INFO(dev_priv)->is_lp)
 #define IS_GEN9_LP(dev_priv)	(IS_GEN9(dev_priv) && IS_LP(dev_priv))
@@ -2871,7 +2926,6 @@ intel_info(const struct drm_i915_private *dev_priv)
 
 #define HWS_NEEDS_PHYSICAL(dev_priv)	((dev_priv)->info.hws_needs_physical)
 
-#define HAS_HW_CONTEXTS(dev_priv)	    ((dev_priv)->info.has_hw_contexts)
 #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
 		((dev_priv)->info.has_logical_ring_contexts)
 #define USES_PPGTT(dev_priv)		(i915.enable_ppgtt)
@@ -2910,6 +2964,7 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define HAS_FW_BLC(dev_priv) 	(INTEL_GEN(dev_priv) > 2)
 #define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr)
 #define HAS_FBC(dev_priv)	((dev_priv)->info.has_fbc)
+#define HAS_CUR_FBC(dev_priv)	(!HAS_GMCH_DISPLAY(dev_priv) && INTEL_INFO(dev_priv)->gen >= 7)
 
 #define HAS_IPS(dev_priv)	(IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
 
@@ -2932,6 +2987,7 @@ intel_info(const struct drm_i915_private *dev_priv)
  * properties, so we have separate macros to test them.
  */
 #define HAS_GUC(dev_priv)	((dev_priv)->info.has_guc)
+#define HAS_GUC_CT(dev_priv)	((dev_priv)->info.has_guc_ct)
 #define HAS_GUC_UCODE(dev_priv)	(HAS_GUC(dev_priv))
 #define HAS_GUC_SCHED(dev_priv)	(HAS_GUC(dev_priv))
 #define HAS_HUC_UCODE(dev_priv)	(HAS_GUC(dev_priv))
@@ -2941,6 +2997,7 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define HAS_POOLED_EU(dev_priv)	((dev_priv)->info.has_pooled_eu)
 
 #define INTEL_PCH_DEVICE_ID_MASK		0xff00
+#define INTEL_PCH_DEVICE_ID_MASK_EXT		0xff80
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE		0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE		0x1c00
 #define INTEL_PCH_PPT_DEVICE_ID_TYPE		0x1e00
@@ -2949,11 +3006,16 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define INTEL_PCH_SPT_DEVICE_ID_TYPE		0xA100
 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE		0x9D00
 #define INTEL_PCH_KBP_DEVICE_ID_TYPE		0xA200
+#define INTEL_PCH_CNP_DEVICE_ID_TYPE		0xA300
+#define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE		0x9D80
 #define INTEL_PCH_P2X_DEVICE_ID_TYPE		0x7100
 #define INTEL_PCH_P3X_DEVICE_ID_TYPE		0x7000
 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE		0x2900 /* qemu q35 has 2918 */
 
 #define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type)
+#define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP)
+#define HAS_PCH_CNP_LP(dev_priv) \
+	((dev_priv)->pch_id == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE)
 #define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP)
 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
@@ -2968,7 +3030,7 @@ intel_info(const struct drm_i915_private *dev_priv)
 
 #define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display)
 
-#define HAS_LSPCON(dev_priv) (IS_GEN9(dev_priv))
+#define HAS_LSPCON(dev_priv) (INTEL_GEN(dev_priv) >= 9)
 
 /* DPF == dynamic parity feature */
 #define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf)
@@ -2978,27 +3040,26 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define GT_FREQUENCY_MULTIPLIER 50
 #define GEN9_FREQ_SCALER 3
 
-#define HAS_DECOUPLED_MMIO(dev_priv) (INTEL_INFO(dev_priv)->has_decoupled_mmio)
-
 #include "i915_trace.h"
 
-static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv)
+static inline bool intel_vtd_active(void)
 {
 #ifdef CONFIG_INTEL_IOMMU
-	if (INTEL_GEN(dev_priv) >= 6 && intel_iommu_gfx_mapped)
+	if (intel_iommu_gfx_mapped)
 		return true;
 #endif
 	return false;
 }
 
+static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv)
+{
+	return INTEL_GEN(dev_priv) >= 6 && intel_vtd_active();
+}
+
 static inline bool
 intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv)
 {
-#ifdef CONFIG_INTEL_IOMMU
-	if (IS_BROXTON(dev_priv) && intel_iommu_gfx_mapped)
-		return true;
-#endif
-	return false;
+	return IS_BROXTON(dev_priv) && intel_vtd_active();
 }
 
 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
@@ -3037,7 +3098,7 @@ extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
 extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
 
-int intel_engines_init_early(struct drm_i915_private *dev_priv);
+int intel_engines_init_mmio(struct drm_i915_private *dev_priv);
 int intel_engines_init(struct drm_i915_private *dev_priv);
 
 /* intel_hotplug.c */
@@ -3074,43 +3135,10 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 		       const char *fmt, ...);
 
 extern void intel_irq_init(struct drm_i915_private *dev_priv);
+extern void intel_irq_fini(struct drm_i915_private *dev_priv);
 int intel_irq_install(struct drm_i915_private *dev_priv);
 void intel_irq_uninstall(struct drm_i915_private *dev_priv);
 
-extern void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
-extern void intel_uncore_init(struct drm_i915_private *dev_priv);
-extern bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
-extern bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
-extern void intel_uncore_fini(struct drm_i915_private *dev_priv);
-extern void intel_uncore_suspend(struct drm_i915_private *dev_priv);
-extern void intel_uncore_resume_early(struct drm_i915_private *dev_priv);
-const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id);
-void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
-				enum forcewake_domains domains);
-void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
-				enum forcewake_domains domains);
-/* Like above but the caller must manage the uncore.lock itself.
- * Must be used with I915_READ_FW and friends.
- */
-void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
-					enum forcewake_domains domains);
-void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
-					enum forcewake_domains domains);
-u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv);
-
-void assert_forcewakes_inactive(struct drm_i915_private *dev_priv);
-
-int intel_wait_for_register(struct drm_i915_private *dev_priv,
-			    i915_reg_t reg,
-			    const u32 mask,
-			    const u32 value,
-			    const unsigned long timeout_ms);
-int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
-			       i915_reg_t reg,
-			       const u32 mask,
-			       const u32 value,
-			       const unsigned long timeout_ms);
-
 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
 {
 	return dev_priv->gvt;
@@ -3208,7 +3236,8 @@ int i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 int i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
-void i915_gem_init_userptr(struct drm_i915_private *dev_priv);
+int i915_gem_init_userptr(struct drm_i915_private *dev_priv);
+void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv);
 int i915_gem_userptr_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file);
 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
@@ -3458,8 +3487,9 @@ int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 #define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX
 
 int __must_check
-i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
-				  bool write);
+i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write);
+int __must_check
+i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write);
 int __must_check
 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
 struct i915_vma * __must_check
@@ -3547,6 +3577,9 @@ i915_gem_context_lookup_timeline(struct i915_gem_context *ctx,
 
 int i915_perf_open_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file);
+void i915_oa_init_reg_state(struct intel_engine_cs *engine,
+			    struct i915_gem_context *ctx,
+			    uint32_t *reg_state);
 
 /* i915_gem_evict.c */
 int __must_check i915_gem_evict_something(struct i915_address_space *vm,
@@ -3557,7 +3590,7 @@ int __must_check i915_gem_evict_something(struct i915_address_space *vm,
 int __must_check i915_gem_evict_for_node(struct i915_address_space *vm,
 					 struct drm_mm_node *node,
 					 unsigned int flags);
-int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
+int i915_gem_evict_vm(struct i915_address_space *vm);
 
 /* belongs in i915_gem_gtt.h */
 static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv)
@@ -3722,8 +3755,8 @@ int  intel_lpe_audio_init(struct drm_i915_private *dev_priv);
 void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv);
 void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv);
 void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
-			    void *eld, int port, int pipe, int tmds_clk_speed,
-			    bool dp_output, int link_rate);
+			    enum pipe pipe, enum port port,
+			    const void *eld, int ls_clock, bool dp_output);
 
 /* intel_i2c.c */
 extern int intel_setup_gmbus(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 615f0a855222..7dcac3bfb771 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -46,15 +46,13 @@
 #include <linux/dma-buf.h>
 
 static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
-static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
-static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
 
 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
 {
-	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
+	if (obj->cache_dirty)
 		return false;
 
-	if (!i915_gem_object_is_coherent(obj))
+	if (!obj->cache_coherent)
 		return true;
 
 	return obj->pin_display;
@@ -145,9 +143,9 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct drm_i915_gem_get_aperture *args = data;
 	struct i915_vma *vma;
-	size_t pinned;
+	u64 pinned;
 
-	pinned = 0;
+	pinned = ggtt->base.reserved;
 	mutex_lock(&dev->struct_mutex);
 	list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 		if (i915_vma_is_pinned(vma))
@@ -235,6 +233,14 @@ err_phys:
 	return st;
 }
 
+static void __start_cpu_write(struct drm_i915_gem_object *obj)
+{
+	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	if (cpu_write_needs_clflush(obj))
+		obj->cache_dirty = true;
+}
+
 static void
 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 				struct sg_table *pages,
@@ -247,11 +253,10 @@ __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 
 	if (needs_clflush &&
 	    (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
-	    !i915_gem_object_is_coherent(obj))
+	    !obj->cache_coherent)
 		drm_clflush_sg(pages);
 
-	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	__start_cpu_write(obj);
 }
 
 static void
@@ -686,6 +691,12 @@ i915_gem_dumb_create(struct drm_file *file,
 			       args->size, &args->handle);
 }
 
+static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
+{
+	return !(obj->cache_level == I915_CACHE_NONE ||
+		 obj->cache_level == I915_CACHE_WT);
+}
+
 /**
  * Creates a new mm object and returns a handle to it.
  * @dev: drm device pointer
@@ -705,6 +716,66 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
 			       args->size, &args->handle);
 }
 
+static inline enum fb_op_origin
+fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
+{
+	return (domain == I915_GEM_DOMAIN_GTT ?
+		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
+}
+
+static void
+flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
+{
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+
+	if (!(obj->base.write_domain & flush_domains))
+		return;
+
+	/* No actual flushing is required for the GTT write domain.  Writes
+	 * to it "immediately" go to main memory as far as we know, so there's
+	 * no chipset flush.  It also doesn't land in render cache.
+	 *
+	 * However, we do have to enforce the order so that all writes through
+	 * the GTT land before any writes to the device, such as updates to
+	 * the GATT itself.
+	 *
+	 * We also have to wait a bit for the writes to land from the GTT.
+	 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
+	 * timing. This issue has only been observed when switching quickly
+	 * between GTT writes and CPU reads from inside the kernel on recent hw,
+	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
+	 * system agents we cannot reproduce this behaviour).
+	 */
+	wmb();
+
+	switch (obj->base.write_domain) {
+	case I915_GEM_DOMAIN_GTT:
+		if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv)) {
+			if (intel_runtime_pm_get_if_in_use(dev_priv)) {
+				spin_lock_irq(&dev_priv->uncore.lock);
+				POSTING_READ_FW(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
+				spin_unlock_irq(&dev_priv->uncore.lock);
+				intel_runtime_pm_put(dev_priv);
+			}
+		}
+
+		intel_fb_obj_flush(obj,
+				   fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
+		break;
+
+	case I915_GEM_DOMAIN_CPU:
+		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
+		break;
+
+	case I915_GEM_DOMAIN_RENDER:
+		if (gpu_write_needs_clflush(obj))
+			obj->cache_dirty = true;
+		break;
+	}
+
+	obj->base.write_domain = 0;
+}
+
 static inline int
 __copy_to_user_swizzled(char __user *cpu_vaddr,
 			const char *gpu_vaddr, int gpu_offset,
@@ -785,8 +856,7 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	if (i915_gem_object_is_coherent(obj) ||
-	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (obj->cache_coherent || !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 		ret = i915_gem_object_set_to_cpu_domain(obj, false);
 		if (ret)
 			goto err_unpin;
@@ -794,14 +864,15 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 			goto out;
 	}
 
-	i915_gem_object_flush_gtt_write_domain(obj);
+	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 
 	/* If we're not in the cpu read domain, set ourself into the gtt
 	 * read domain and manually flush cachelines (if required). This
 	 * optimizes for the case when the gpu will dirty the data
 	 * anyway again before the next pread happens.
 	 */
-	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
+	if (!obj->cache_dirty &&
+	    !(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
 		*needs_clflush = CLFLUSH_BEFORE;
 
 out:
@@ -837,8 +908,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	if (i915_gem_object_is_coherent(obj) ||
-	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (obj->cache_coherent || !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 		ret = i915_gem_object_set_to_cpu_domain(obj, true);
 		if (ret)
 			goto err_unpin;
@@ -846,21 +916,23 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 			goto out;
 	}
 
-	i915_gem_object_flush_gtt_write_domain(obj);
+	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 
 	/* If we're not in the cpu write domain, set ourself into the
 	 * gtt write domain and manually flush cachelines (as required).
 	 * This optimizes for the case when the gpu will use the data
 	 * right away and we therefore have to clflush anyway.
 	 */
-	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
+	if (!obj->cache_dirty) {
 		*needs_clflush |= CLFLUSH_AFTER;
 
-	/* Same trick applies to invalidate partially written cachelines read
-	 * before writing.
-	 */
-	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
-		*needs_clflush |= CLFLUSH_BEFORE;
+		/*
+		 * Same trick applies to invalidate partially written
+		 * cachelines read before writing.
+		 */
+		if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
+			*needs_clflush |= CLFLUSH_BEFORE;
+	}
 
 out:
 	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
@@ -1501,13 +1573,6 @@ err:
 	return ret;
 }
 
-static inline enum fb_op_origin
-write_origin(struct drm_i915_gem_object *obj, unsigned domain)
-{
-	return (domain == I915_GEM_DOMAIN_GTT ?
-		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
-}
-
 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *i915;
@@ -1591,10 +1656,12 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	if (err)
 		goto out_unpin;
 
-	if (read_domains & I915_GEM_DOMAIN_GTT)
-		err = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
+	if (read_domains & I915_GEM_DOMAIN_WC)
+		err = i915_gem_object_set_to_wc_domain(obj, write_domain);
+	else if (read_domains & I915_GEM_DOMAIN_GTT)
+		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
 	else
-		err = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
+		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
 
 	/* And bump the LRU for this access */
 	i915_gem_object_bump_inactive_ggtt(obj);
@@ -1602,7 +1669,8 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	mutex_unlock(&dev->struct_mutex);
 
 	if (write_domain != 0)
-		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
+		intel_fb_obj_invalidate(obj,
+					fb_write_origin(obj, write_domain));
 
 out_unpin:
 	i915_gem_object_unpin_pages(obj);
@@ -1737,6 +1805,9 @@ static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
  *     into userspace. (This view is aligned and sized appropriately for
  *     fenced access.)
  *
+ * 2 - Recognise WC as a separate cache domain so that we can flush the
+ *     delayed writes via GTT before performing direct access via WC.
+ *
  * Restrictions:
  *
  *  * snoopable objects cannot be accessed via the GTT. It can cause machine
@@ -1764,7 +1835,7 @@ static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
  */
 int i915_gem_mmap_gtt_version(void)
 {
-	return 1;
+	return 2;
 }
 
 static inline struct i915_ggtt_view
@@ -2228,7 +2299,7 @@ void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
 	if (obj->mm.mapping) {
 		void *ptr;
 
-		ptr = ptr_mask_bits(obj->mm.mapping);
+		ptr = page_mask_bits(obj->mm.mapping);
 		if (is_vmalloc_addr(ptr))
 			vunmap(ptr);
 		else
@@ -2315,8 +2386,7 @@ rebuild_st:
 	 * Fail silently without starting the shrinker
 	 */
 	mapping = obj->base.filp->f_mapping;
-	noreclaim = mapping_gfp_constraint(mapping,
-					   ~(__GFP_IO | __GFP_RECLAIM));
+	noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
 	noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
 
 	sg = st->sgl;
@@ -2524,7 +2594,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
 
 	if (n_pages > ARRAY_SIZE(stack_pages)) {
 		/* Too big for stack -- allocate temporary array instead */
-		pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
+		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_TEMPORARY);
 		if (!pages)
 			return NULL;
 	}
@@ -2546,7 +2616,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
 	addr = vmap(pages, n_pages, 0, pgprot);
 
 	if (pages != stack_pages)
-		drm_free_large(pages);
+		kvfree(pages);
 
 	return addr;
 }
@@ -2580,7 +2650,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 	}
 	GEM_BUG_ON(!obj->mm.pages);
 
-	ptr = ptr_unpack_bits(obj->mm.mapping, has_type);
+	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
 	if (ptr && has_type != type) {
 		if (pinned) {
 			ret = -EBUSY;
@@ -2602,7 +2672,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 			goto err_unpin;
 		}
 
-		obj->mm.mapping = ptr_pack_bits(ptr, type);
+		obj->mm.mapping = page_pack_bits(ptr, type);
 	}
 
 out_unlock:
@@ -2987,12 +3057,14 @@ static void engine_set_wedged(struct intel_engine_cs *engine)
 	 */
 
 	if (i915.enable_execlists) {
+		struct execlist_port *port = engine->execlist_port;
 		unsigned long flags;
+		unsigned int n;
 
 		spin_lock_irqsave(&engine->timeline->lock, flags);
 
-		i915_gem_request_put(engine->execlist_port[0].request);
-		i915_gem_request_put(engine->execlist_port[1].request);
+		for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
+			i915_gem_request_put(port_request(&port[n]));
 		memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
 		engine->execlist_queue = RB_ROOT;
 		engine->execlist_first = NULL;
@@ -3121,8 +3193,6 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	struct drm_i915_private *dev_priv =
 		container_of(work, typeof(*dev_priv), gt.idle_work.work);
 	struct drm_device *dev = &dev_priv->drm;
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
 	bool rearm_hangcheck;
 
 	if (!READ_ONCE(dev_priv->gt.awake))
@@ -3160,10 +3230,8 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	if (wait_for(intel_engines_are_idle(dev_priv), 10))
 		DRM_ERROR("Timeout waiting for engines to idle\n");
 
-	for_each_engine(engine, dev_priv, id) {
-		intel_engine_disarm_breadcrumbs(engine);
-		i915_gem_batch_pool_fini(&engine->batch_pool);
-	}
+	intel_engines_mark_idle(dev_priv);
+	i915_gem_timelines_mark_idle(dev_priv);
 
 	GEM_BUG_ON(!dev_priv->gt.awake);
 	dev_priv->gt.awake = false;
@@ -3193,6 +3261,10 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 		if (vma->vm->file == fpriv)
 			i915_vma_close(vma);
 
+	vma = obj->vma_hashed;
+	if (vma && vma->ctx->file_priv == fpriv)
+		i915_vma_unlink_ctx(vma);
+
 	if (i915_gem_object_is_active(obj) &&
 	    !i915_gem_object_has_active_reference(obj)) {
 		i915_gem_object_set_active_reference(obj);
@@ -3344,73 +3416,89 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
 	return ret;
 }
 
-/** Flushes the GTT write domain for the object if it's dirty. */
-static void
-i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
+static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-
-	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
-		return;
-
-	/* No actual flushing is required for the GTT write domain.  Writes
-	 * to it "immediately" go to main memory as far as we know, so there's
-	 * no chipset flush.  It also doesn't land in render cache.
-	 *
-	 * However, we do have to enforce the order so that all writes through
-	 * the GTT land before any writes to the device, such as updates to
-	 * the GATT itself.
-	 *
-	 * We also have to wait a bit for the writes to land from the GTT.
-	 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
-	 * timing. This issue has only been observed when switching quickly
-	 * between GTT writes and CPU reads from inside the kernel on recent hw,
-	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
-	 * system agents we cannot reproduce this behaviour).
+	/*
+	 * We manually flush the CPU domain so that we can override and
+	 * force the flush for the display, and perform it asyncrhonously.
 	 */
-	wmb();
-	if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv)) {
-		if (intel_runtime_pm_get_if_in_use(dev_priv)) {
-			spin_lock_irq(&dev_priv->uncore.lock);
-			POSTING_READ_FW(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
-			spin_unlock_irq(&dev_priv->uncore.lock);
-			intel_runtime_pm_put(dev_priv);
-		}
-	}
-
-	intel_fb_obj_flush(obj, write_origin(obj, I915_GEM_DOMAIN_GTT));
-
+	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
+	if (obj->cache_dirty)
+		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
 	obj->base.write_domain = 0;
 }
 
-/** Flushes the CPU write domain for the object if it's dirty. */
-static void
-i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
+void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
 {
-	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
+	if (!READ_ONCE(obj->pin_display))
 		return;
 
-	i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
-	obj->base.write_domain = 0;
+	mutex_lock(&obj->base.dev->struct_mutex);
+	__i915_gem_object_flush_for_display(obj);
+	mutex_unlock(&obj->base.dev->struct_mutex);
 }
 
-static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
+/**
+ * Moves a single object to the WC read, and possibly write domain.
+ * @obj: object to act on
+ * @write: ask for write access or read only
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+int
+i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
 {
-	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU && !obj->cache_dirty)
-		return;
+	int ret;
 
-	i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
-	obj->base.write_domain = 0;
-}
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
-{
-	if (!READ_ONCE(obj->pin_display))
-		return;
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED |
+				   (write ? I915_WAIT_ALL : 0),
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
+	if (ret)
+		return ret;
 
-	mutex_lock(&obj->base.dev->struct_mutex);
-	__i915_gem_object_flush_for_display(obj);
-	mutex_unlock(&obj->base.dev->struct_mutex);
+	if (obj->base.write_domain == I915_GEM_DOMAIN_WC)
+		return 0;
+
+	/* Flush and acquire obj->pages so that we are coherent through
+	 * direct access in memory with previous cached writes through
+	 * shmemfs and that our cache domain tracking remains valid.
+	 * For example, if the obj->filp was moved to swap without us
+	 * being notified and releasing the pages, we would mistakenly
+	 * continue to assume that the obj remained out of the CPU cached
+	 * domain.
+	 */
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		return ret;
+
+	flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
+
+	/* Serialise direct access to this object with the barriers for
+	 * coherent writes from the GPU, by effectively invalidating the
+	 * WC domain upon first access.
+	 */
+	if ((obj->base.read_domains & I915_GEM_DOMAIN_WC) == 0)
+		mb();
+
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_WC) != 0);
+	obj->base.read_domains |= I915_GEM_DOMAIN_WC;
+	if (write) {
+		obj->base.read_domains = I915_GEM_DOMAIN_WC;
+		obj->base.write_domain = I915_GEM_DOMAIN_WC;
+		obj->mm.dirty = true;
+	}
+
+	i915_gem_object_unpin_pages(obj);
+	return 0;
 }
 
 /**
@@ -3452,7 +3540,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	if (ret)
 		return ret;
 
-	i915_gem_object_flush_cpu_write_domain(obj);
+	flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
 
 	/* Serialise direct access to this object with the barriers for
 	 * coherent writes from the GPU, by effectively invalidating the
@@ -3595,13 +3683,11 @@ restart:
 		}
 	}
 
-	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU &&
-	    i915_gem_object_is_coherent(obj))
-		obj->cache_dirty = true;
-
 	list_for_each_entry(vma, &obj->vma_list, obj_link)
 		vma->node.color = cache_level;
 	obj->cache_level = cache_level;
+	obj->cache_coherent = i915_gem_object_is_coherent(obj);
+	obj->cache_dirty = true; /* Always invalidate stale cachelines */
 
 	return 0;
 }
@@ -3823,10 +3909,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	if (ret)
 		return ret;
 
-	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
-		return 0;
-
-	i915_gem_object_flush_gtt_write_domain(obj);
+	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 
 	/* Flush the CPU cache if it's still invalid. */
 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
@@ -3837,15 +3920,13 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+	GEM_BUG_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
 
 	/* If we're writing through the CPU, then the GPU read domains will
 	 * need to be invalidated at next use.
 	 */
-	if (write) {
-		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
+	if (write)
+		__start_cpu_write(obj);
 
 	return 0;
 }
@@ -4020,7 +4101,7 @@ __busy_set_if_active(const struct dma_fence *fence,
 	if (i915_gem_request_completed(rq))
 		return 0;
 
-	return flag(rq->engine->exec_id);
+	return flag(rq->engine->uabi_id);
 }
 
 static __always_inline unsigned int
@@ -4177,7 +4258,6 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
 	INIT_LIST_HEAD(&obj->global_link);
 	INIT_LIST_HEAD(&obj->userfault_link);
-	INIT_LIST_HEAD(&obj->obj_exec_link);
 	INIT_LIST_HEAD(&obj->vma_list);
 	INIT_LIST_HEAD(&obj->batch_pool_link);
 
@@ -4219,7 +4299,7 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
 	 * catch if we ever need to fix it. In the meantime, if you do spot
 	 * such a local variable, please consider fixing!
 	 */
-	if (WARN_ON(size >> PAGE_SHIFT > INT_MAX))
+	if (size >> PAGE_SHIFT > INT_MAX)
 		return ERR_PTR(-E2BIG);
 
 	if (overflows_type(size, obj->base.size))
@@ -4266,6 +4346,9 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
 	} else
 		obj->cache_level = I915_CACHE_NONE;
 
+	obj->cache_coherent = i915_gem_object_is_coherent(obj);
+	obj->cache_dirty = !obj->cache_coherent;
+
 	trace_i915_gem_object_create(obj);
 
 	return obj;
@@ -4314,7 +4397,6 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
 		GEM_BUG_ON(i915_gem_object_is_active(obj));
 		list_for_each_entry_safe(vma, vn,
 					 &obj->vma_list, obj_link) {
-			GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 			GEM_BUG_ON(i915_vma_is_active(vma));
 			vma->flags &= ~I915_VMA_PIN_MASK;
 			i915_vma_close(vma);
@@ -4327,6 +4409,8 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
 	intel_runtime_pm_put(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
+	cond_resched();
+
 	llist_for_each_entry_safe(obj, on, freed, freed) {
 		GEM_BUG_ON(obj->bind_count);
 		GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
@@ -4374,8 +4458,11 @@ static void __i915_gem_free_work(struct work_struct *work)
 	 * unbound now.
 	 */
 
-	while ((freed = llist_del_all(&i915->mm.free_list)))
+	while ((freed = llist_del_all(&i915->mm.free_list))) {
 		__i915_gem_free_objects(i915, freed);
+		if (need_resched())
+			break;
+	}
 }
 
 static void __i915_gem_free_object_rcu(struct rcu_head *head)
@@ -4440,10 +4527,9 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
 	 * try to take over. The only way to remove the earlier state
 	 * is by resetting. However, resetting on earlier gen is tricky as
 	 * it may impact the display and we are uncertain about the stability
-	 * of the reset, so we only reset recent machines with logical
-	 * context support (that must be reset to remove any stray contexts).
+	 * of the reset, so this could be applied to even earlier gen.
 	 */
-	if (HAS_HW_CONTEXTS(i915)) {
+	if (INTEL_GEN(i915) >= 5) {
 		int reset = intel_gpu_reset(i915, ALL_ENGINES);
 		WARN_ON(reset && reset != -ENODEV);
 	}
@@ -4686,11 +4772,9 @@ bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
 	if (value >= 0)
 		return value;
 
-#ifdef CONFIG_INTEL_IOMMU
 	/* Enable semaphores on SNB when IO remapping is off */
-	if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
+	if (IS_GEN6(dev_priv) && intel_vtd_active())
 		return false;
-#endif
 
 	return true;
 }
@@ -4701,7 +4785,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
 
-	i915_gem_clflush_init(dev_priv);
+	dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
 
 	if (!i915.enable_execlists) {
 		dev_priv->gt.resume = intel_legacy_submission_resume;
@@ -4719,7 +4803,9 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	 */
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	i915_gem_init_userptr(dev_priv);
+	ret = i915_gem_init_userptr(dev_priv);
+	if (ret)
+		goto out_unlock;
 
 	ret = i915_gem_init_ggtt(dev_priv);
 	if (ret)
@@ -4824,12 +4910,16 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	if (!dev_priv->dependencies)
 		goto err_requests;
 
+	dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
+	if (!dev_priv->priorities)
+		goto err_dependencies;
+
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	INIT_LIST_HEAD(&dev_priv->gt.timelines);
 	err = i915_gem_timeline_init__global(dev_priv);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 	if (err)
-		goto err_dependencies;
+		goto err_priorities;
 
 	INIT_LIST_HEAD(&dev_priv->context_list);
 	INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
@@ -4847,14 +4937,14 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 
 	init_waitqueue_head(&dev_priv->pending_flip_queue);
 
-	dev_priv->mm.interruptible = true;
-
 	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
 
 	spin_lock_init(&dev_priv->fb_tracking.lock);
 
 	return 0;
 
+err_priorities:
+	kmem_cache_destroy(dev_priv->priorities);
 err_dependencies:
 	kmem_cache_destroy(dev_priv->dependencies);
 err_requests:
@@ -4878,6 +4968,7 @@ void i915_gem_load_cleanup(struct drm_i915_private *dev_priv)
 	WARN_ON(!list_empty(&dev_priv->gt.timelines));
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
+	kmem_cache_destroy(dev_priv->priorities);
 	kmem_cache_destroy(dev_priv->dependencies);
 	kmem_cache_destroy(dev_priv->requests);
 	kmem_cache_destroy(dev_priv->vmas);
@@ -4889,9 +4980,10 @@ void i915_gem_load_cleanup(struct drm_i915_private *dev_priv)
 
 int i915_gem_freeze(struct drm_i915_private *dev_priv)
 {
-	mutex_lock(&dev_priv->drm.struct_mutex);
+	/* Discard all purgeable objects, let userspace recover those as
+	 * required after resuming.
+	 */
 	i915_gem_shrink_all(dev_priv);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	return 0;
 }
@@ -4916,17 +5008,16 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 	 * we update that state just before writing out the image.
 	 *
 	 * To try and reduce the hibernation image, we manually shrink
-	 * the objects as well.
+	 * the objects as well, see i915_gem_freeze()
 	 */
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
 	i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
+	i915_gem_drain_freed_objects(dev_priv);
 
+	mutex_lock(&dev_priv->drm.struct_mutex);
 	for (p = phases; *p; p++) {
-		list_for_each_entry(obj, *p, global_link) {
-			obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-			obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-		}
+		list_for_each_entry(obj, *p, global_link)
+			__start_cpu_write(obj);
 	}
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 5a49487368ca..ee54597465b6 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -25,6 +25,8 @@
 #ifndef __I915_GEM_H__
 #define __I915_GEM_H__
 
+#include <linux/bug.h>
+
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
 #define GEM_BUG_ON(expr) BUG_ON(expr)
 #define GEM_WARN_ON(expr) WARN_ON(expr)
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index 41aa598c4f3b..c93005c2e0fb 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -114,12 +114,27 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 	list_for_each_entry(obj, list, batch_pool_link) {
 		/* The batches are strictly LRU ordered */
 		if (i915_gem_object_is_active(obj)) {
-			if (!reservation_object_test_signaled_rcu(obj->resv,
-								  true))
+			struct reservation_object *resv = obj->resv;
+
+			if (!reservation_object_test_signaled_rcu(resv, true))
 				break;
 
 			i915_gem_retire_requests(pool->engine->i915);
 			GEM_BUG_ON(i915_gem_object_is_active(obj));
+
+			/*
+			 * The object is now idle, clear the array of shared
+			 * fences before we add a new request. Although, we
+			 * remain on the same engine, we may be on a different
+			 * timeline and so may continually grow the array,
+			 * trapping a reference to all the old fences, rather
+			 * than replace the existing fence.
+			 */
+			if (rcu_access_pointer(resv->fence)) {
+				reservation_object_lock(resv, NULL);
+				reservation_object_add_excl_fence(resv, NULL);
+				reservation_object_unlock(resv);
+			}
 		}
 
 		GEM_BUG_ON(!reservation_object_test_signaled_rcu(obj->resv,
diff --git a/drivers/gpu/drm/i915/i915_gem_clflush.c b/drivers/gpu/drm/i915/i915_gem_clflush.c
index ffd01e02fe94..152f16c11878 100644
--- a/drivers/gpu/drm/i915/i915_gem_clflush.c
+++ b/drivers/gpu/drm/i915/i915_gem_clflush.c
@@ -27,7 +27,6 @@
 #include "i915_gem_clflush.h"
 
 static DEFINE_SPINLOCK(clflush_lock);
-static u64 clflush_context;
 
 struct clflush {
 	struct dma_fence dma; /* Must be first for dma_fence_free() */
@@ -72,8 +71,6 @@ static const struct dma_fence_ops i915_clflush_ops = {
 static void __i915_do_clflush(struct drm_i915_gem_object *obj)
 {
 	drm_clflush_sg(obj->mm.pages);
-	obj->cache_dirty = false;
-
 	intel_fb_obj_flush(obj, ORIGIN_CPU);
 }
 
@@ -82,9 +79,6 @@ static void i915_clflush_work(struct work_struct *work)
 	struct clflush *clflush = container_of(work, typeof(*clflush), work);
 	struct drm_i915_gem_object *obj = clflush->obj;
 
-	if (!obj->cache_dirty)
-		goto out;
-
 	if (i915_gem_object_pin_pages(obj)) {
 		DRM_ERROR("Failed to acquire obj->pages for clflushing\n");
 		goto out;
@@ -132,10 +126,10 @@ void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 	 * anything not backed by physical memory we consider to be always
 	 * coherent and not need clflushing.
 	 */
-	if (!i915_gem_object_has_struct_page(obj))
+	if (!i915_gem_object_has_struct_page(obj)) {
+		obj->cache_dirty = false;
 		return;
-
-	obj->cache_dirty = true;
+	}
 
 	/* If the GPU is snooping the contents of the CPU cache,
 	 * we do not need to manually clear the CPU cache lines.  However,
@@ -145,7 +139,7 @@ void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 	 * snooping behaviour occurs naturally as the result of our domain
 	 * tracking.
 	 */
-	if (!(flags & I915_CLFLUSH_FORCE) && i915_gem_object_is_coherent(obj))
+	if (!(flags & I915_CLFLUSH_FORCE) && obj->cache_coherent)
 		return;
 
 	trace_i915_gem_object_clflush(obj);
@@ -154,10 +148,12 @@ void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 	if (!(flags & I915_CLFLUSH_SYNC))
 		clflush = kmalloc(sizeof(*clflush), GFP_KERNEL);
 	if (clflush) {
+		GEM_BUG_ON(!obj->cache_dirty);
+
 		dma_fence_init(&clflush->dma,
 			       &i915_clflush_ops,
 			       &clflush_lock,
-			       clflush_context,
+			       to_i915(obj->base.dev)->mm.unordered_timeline,
 			       0);
 		i915_sw_fence_init(&clflush->wait, i915_clflush_notify);
 
@@ -181,9 +177,6 @@ void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 	} else {
 		GEM_BUG_ON(obj->base.write_domain != I915_GEM_DOMAIN_CPU);
 	}
-}
 
-void i915_gem_clflush_init(struct drm_i915_private *i915)
-{
-	clflush_context = dma_fence_context_alloc(1);
+	obj->cache_dirty = false;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_clflush.h b/drivers/gpu/drm/i915/i915_gem_clflush.h
index b62d61a2d15f..2455a7820937 100644
--- a/drivers/gpu/drm/i915/i915_gem_clflush.h
+++ b/drivers/gpu/drm/i915/i915_gem_clflush.h
@@ -28,7 +28,6 @@
 struct drm_i915_private;
 struct drm_i915_gem_object;
 
-void i915_gem_clflush_init(struct drm_i915_private *i915);
 void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 			     unsigned int flags);
 #define I915_CLFLUSH_FORCE BIT(0)
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 8bd0c4966913..39ed58a21fc1 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -85,6 +85,7 @@
  *
  */
 
+#include <linux/log2.h>
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
@@ -92,31 +93,69 @@
 
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
-static int get_context_size(struct drm_i915_private *dev_priv)
+/* Initial size (as log2) to preallocate the handle->object hashtable */
+#define VMA_HT_BITS 2u /* 4 x 2 pointers, 64 bytes minimum */
+
+static void resize_vma_ht(struct work_struct *work)
 {
-	int ret;
-	u32 reg;
+	struct i915_gem_context_vma_lut *lut =
+		container_of(work, typeof(*lut), resize);
+	unsigned int bits, new_bits, size, i;
+	struct hlist_head *new_ht;
+
+	GEM_BUG_ON(!(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS));
+
+	bits = 1 + ilog2(4*lut->ht_count/3 + 1);
+	new_bits = min_t(unsigned int,
+			 max(bits, VMA_HT_BITS),
+			 sizeof(unsigned int) * BITS_PER_BYTE - 1);
+	if (new_bits == lut->ht_bits)
+		goto out;
 
-	switch (INTEL_GEN(dev_priv)) {
-	case 6:
-		reg = I915_READ(CXT_SIZE);
-		ret = GEN6_CXT_TOTAL_SIZE(reg) * 64;
-		break;
-	case 7:
-		reg = I915_READ(GEN7_CXT_SIZE);
-		if (IS_HASWELL(dev_priv))
-			ret = HSW_CXT_TOTAL_SIZE;
-		else
-			ret = GEN7_CXT_TOTAL_SIZE(reg) * 64;
-		break;
-	case 8:
-		ret = GEN8_CXT_TOTAL_SIZE;
-		break;
-	default:
-		BUG();
+	new_ht = kzalloc(sizeof(*new_ht)<<new_bits, GFP_KERNEL | __GFP_NOWARN);
+	if (!new_ht)
+		new_ht = vzalloc(sizeof(*new_ht)<<new_bits);
+	if (!new_ht)
+		/* Pretend resize succeeded and stop calling us for a bit! */
+		goto out;
+
+	size = BIT(lut->ht_bits);
+	for (i = 0; i < size; i++) {
+		struct i915_vma *vma;
+		struct hlist_node *tmp;
+
+		hlist_for_each_entry_safe(vma, tmp, &lut->ht[i], ctx_node)
+			hlist_add_head(&vma->ctx_node,
+				       &new_ht[hash_32(vma->ctx_handle,
+						       new_bits)]);
 	}
+	kvfree(lut->ht);
+	lut->ht = new_ht;
+	lut->ht_bits = new_bits;
+out:
+	smp_store_release(&lut->ht_size, BIT(bits));
+	GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
+}
 
-	return ret;
+static void vma_lut_free(struct i915_gem_context *ctx)
+{
+	struct i915_gem_context_vma_lut *lut = &ctx->vma_lut;
+	unsigned int i, size;
+
+	if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS)
+		cancel_work_sync(&lut->resize);
+
+	size = BIT(lut->ht_bits);
+	for (i = 0; i < size; i++) {
+		struct i915_vma *vma;
+
+		hlist_for_each_entry(vma, &lut->ht[i], ctx_node) {
+			vma->obj->vma_hashed = NULL;
+			vma->ctx = NULL;
+			i915_vma_put(vma);
+		}
+	}
+	kvfree(lut->ht);
 }
 
 void i915_gem_context_free(struct kref *ctx_ref)
@@ -128,6 +167,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
 	trace_i915_context_free(ctx);
 	GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
 
+	vma_lut_free(ctx);
 	i915_ppgtt_put(ctx->ppgtt);
 
 	for (i = 0; i < I915_NUM_ENGINES; i++) {
@@ -145,51 +185,13 @@ void i915_gem_context_free(struct kref *ctx_ref)
 
 	kfree(ctx->name);
 	put_pid(ctx->pid);
+
 	list_del(&ctx->link);
 
 	ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id);
 	kfree(ctx);
 }
 
-static struct drm_i915_gem_object *
-alloc_context_obj(struct drm_i915_private *dev_priv, u64 size)
-{
-	struct drm_i915_gem_object *obj;
-	int ret;
-
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
-	obj = i915_gem_object_create(dev_priv, size);
-	if (IS_ERR(obj))
-		return obj;
-
-	/*
-	 * Try to make the context utilize L3 as well as LLC.
-	 *
-	 * On VLV we don't have L3 controls in the PTEs so we
-	 * shouldn't touch the cache level, especially as that
-	 * would make the object snooped which might have a
-	 * negative performance impact.
-	 *
-	 * Snooping is required on non-llc platforms in execlist
-	 * mode, but since all GGTT accesses use PAT entry 0 we
-	 * get snooping anyway regardless of cache_level.
-	 *
-	 * This is only applicable for Ivy Bridge devices since
-	 * later platforms don't have L3 control bits in the PTE.
-	 */
-	if (IS_IVYBRIDGE(dev_priv)) {
-		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
-		/* Failure shouldn't ever happen this early */
-		if (WARN_ON(ret)) {
-			i915_gem_object_put(obj);
-			return ERR_PTR(ret);
-		}
-	}
-
-	return obj;
-}
-
 static void context_close(struct i915_gem_context *ctx)
 {
 	i915_gem_context_set_closed(ctx);
@@ -265,26 +267,18 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 	kref_init(&ctx->ref);
 	list_add_tail(&ctx->link, &dev_priv->context_list);
 	ctx->i915 = dev_priv;
+	ctx->priority = I915_PRIORITY_NORMAL;
 
-	if (dev_priv->hw_context_size) {
-		struct drm_i915_gem_object *obj;
-		struct i915_vma *vma;
+	ctx->vma_lut.ht_bits = VMA_HT_BITS;
+	ctx->vma_lut.ht_size = BIT(VMA_HT_BITS);
+	BUILD_BUG_ON(BIT(VMA_HT_BITS) == I915_CTX_RESIZE_IN_PROGRESS);
+	ctx->vma_lut.ht = kcalloc(ctx->vma_lut.ht_size,
+				  sizeof(*ctx->vma_lut.ht),
+				  GFP_KERNEL);
+	if (!ctx->vma_lut.ht)
+		goto err_out;
 
-		obj = alloc_context_obj(dev_priv, dev_priv->hw_context_size);
-		if (IS_ERR(obj)) {
-			ret = PTR_ERR(obj);
-			goto err_out;
-		}
-
-		vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL);
-		if (IS_ERR(vma)) {
-			i915_gem_object_put(obj);
-			ret = PTR_ERR(vma);
-			goto err_out;
-		}
-
-		ctx->engine[RCS].state = vma;
-	}
+	INIT_WORK(&ctx->vma_lut.resize, resize_vma_ht);
 
 	/* Default context will never have a file_priv */
 	ret = DEFAULT_CONTEXT_HANDLE;
@@ -292,7 +286,7 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 		ret = idr_alloc(&file_priv->context_idr, ctx,
 				DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL);
 		if (ret < 0)
-			goto err_out;
+			goto err_lut;
 	}
 	ctx->user_handle = ret;
 
@@ -333,6 +327,8 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 err_pid:
 	put_pid(ctx->pid);
 	idr_remove(&file_priv->context_idr, ctx->user_handle);
+err_lut:
+	kvfree(ctx->vma_lut.ht);
 err_out:
 	context_close(ctx);
 	return ERR_PTR(ret);
@@ -443,21 +439,6 @@ int i915_gem_context_init(struct drm_i915_private *dev_priv)
 	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX);
 	ida_init(&dev_priv->context_hw_ida);
 
-	if (i915.enable_execlists) {
-		/* NB: intentionally left blank. We will allocate our own
-		 * backing objects as we need them, thank you very much */
-		dev_priv->hw_context_size = 0;
-	} else if (HAS_HW_CONTEXTS(dev_priv)) {
-		dev_priv->hw_context_size =
-			round_up(get_context_size(dev_priv),
-				 I915_GTT_PAGE_SIZE);
-		if (dev_priv->hw_context_size > (1<<20)) {
-			DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size %d\n",
-					 dev_priv->hw_context_size);
-			dev_priv->hw_context_size = 0;
-		}
-	}
-
 	ctx = i915_gem_create_context(dev_priv, NULL);
 	if (IS_ERR(ctx)) {
 		DRM_ERROR("Failed to create default global context (error %ld)\n",
@@ -477,8 +458,8 @@ int i915_gem_context_init(struct drm_i915_private *dev_priv)
 	GEM_BUG_ON(!i915_gem_context_is_kernel(ctx));
 
 	DRM_DEBUG_DRIVER("%s context support initialized\n",
-			i915.enable_execlists ? "LR" :
-			dev_priv->hw_context_size ? "HW" : "fake");
+			 dev_priv->engine[RCS]->context_size ? "logical" :
+			 "fake");
 	return 0;
 }
 
@@ -941,11 +922,6 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
-static bool contexts_enabled(struct drm_device *dev)
-{
-	return i915.enable_execlists || to_i915(dev)->hw_context_size;
-}
-
 static bool client_is_banned(struct drm_i915_file_private *file_priv)
 {
 	return file_priv->context_bans > I915_MAX_CLIENT_CONTEXT_BANS;
@@ -954,12 +930,13 @@ static bool client_is_banned(struct drm_i915_file_private *file_priv)
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
 				  struct drm_file *file)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_context_create *args = data;
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 	struct i915_gem_context *ctx;
 	int ret;
 
-	if (!contexts_enabled(dev))
+	if (!dev_priv->engine[RCS]->context_size)
 		return -ENODEV;
 
 	if (args->pad != 0)
@@ -977,7 +954,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
 	if (ret)
 		return ret;
 
-	ctx = i915_gem_create_context(to_i915(dev), file_priv);
+	ctx = i915_gem_create_context(dev_priv, file_priv);
 	mutex_unlock(&dev->struct_mutex);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
@@ -1138,9 +1115,6 @@ int i915_gem_context_reset_stats_ioctl(struct drm_device *dev,
 	if (args->flags || args->pad)
 		return -EINVAL;
 
-	if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN))
-		return -EPERM;
-
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index 4af2ab94558b..82c99ba92ad3 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -143,6 +143,32 @@ struct i915_gem_context {
 	/** ggtt_offset_bias: placement restriction for context objects */
 	u32 ggtt_offset_bias;
 
+	struct i915_gem_context_vma_lut {
+		/** ht_size: last request size to allocate the hashtable for. */
+		unsigned int ht_size;
+#define I915_CTX_RESIZE_IN_PROGRESS BIT(0)
+		/** ht_bits: real log2(size) of hashtable. */
+		unsigned int ht_bits;
+		/** ht_count: current number of entries inside the hashtable */
+		unsigned int ht_count;
+
+		/** ht: the array of buckets comprising the simple hashtable */
+		struct hlist_head *ht;
+
+		/**
+		 * resize: After an execbuf completes, we check the load factor
+		 * of the hashtable. If the hashtable is too full, or too empty,
+		 * we schedule a task to resize the hashtable. During the
+		 * resize, the entries are moved between different buckets and
+		 * so we cannot simultaneously read the hashtable as it is
+		 * being resized (unlike rhashtable). Therefore we treat the
+		 * active work as a strong barrier, pausing a subsequent
+		 * execbuf to wait for the resize worker to complete, if
+		 * required.
+		 */
+		struct work_struct resize;
+	} vma_lut;
+
 	/** engine: per-engine logical HW state */
 	struct intel_context {
 		struct i915_vma *state;
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
index f225bf680b6d..6176e589cf09 100644
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
@@ -122,12 +122,36 @@ static void i915_gem_dmabuf_kunmap_atomic(struct dma_buf *dma_buf, unsigned long
 }
 static void *i915_gem_dmabuf_kmap(struct dma_buf *dma_buf, unsigned long page_num)
 {
+	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
+	struct page *page;
+
+	if (page_num >= obj->base.size >> PAGE_SHIFT)
+		return NULL;
+
+	if (!i915_gem_object_has_struct_page(obj))
+		return NULL;
+
+	if (i915_gem_object_pin_pages(obj))
+		return NULL;
+
+	/* Synchronisation is left to the caller (via .begin_cpu_access()) */
+	page = i915_gem_object_get_page(obj, page_num);
+	if (IS_ERR(page))
+		goto err_unpin;
+
+	return kmap(page);
+
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
 	return NULL;
 }
 
 static void i915_gem_dmabuf_kunmap(struct dma_buf *dma_buf, unsigned long page_num, void *addr)
 {
+	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
 
+	kunmap(virt_to_page(addr));
+	i915_gem_object_unpin_pages(obj);
 }
 
 static int i915_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 51e365f70464..a193f1b36c67 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -50,6 +50,29 @@ static bool ggtt_is_idle(struct drm_i915_private *dev_priv)
 	return true;
 }
 
+static int ggtt_flush(struct drm_i915_private *i915)
+{
+	int err;
+
+	/* Not everything in the GGTT is tracked via vma (otherwise we
+	 * could evict as required with minimal stalling) so we are forced
+	 * to idle the GPU and explicitly retire outstanding requests in
+	 * the hopes that we can then remove contexts and the like only
+	 * bound by their active reference.
+	 */
+	err = i915_gem_switch_to_kernel_context(i915);
+	if (err)
+		return err;
+
+	err = i915_gem_wait_for_idle(i915,
+				     I915_WAIT_INTERRUPTIBLE |
+				     I915_WAIT_LOCKED);
+	if (err)
+		return err;
+
+	return 0;
+}
+
 static bool
 mark_free(struct drm_mm_scan *scan,
 	  struct i915_vma *vma,
@@ -59,13 +82,10 @@ mark_free(struct drm_mm_scan *scan,
 	if (i915_vma_is_pinned(vma))
 		return false;
 
-	if (WARN_ON(!list_empty(&vma->exec_list)))
-		return false;
-
 	if (flags & PIN_NONFAULT && !list_empty(&vma->obj->userfault_link))
 		return false;
 
-	list_add(&vma->exec_list, unwind);
+	list_add(&vma->evict_link, unwind);
 	return drm_mm_scan_add_block(scan, &vma->node);
 }
 
@@ -157,11 +177,9 @@ search_again:
 	} while (*++phase);
 
 	/* Nothing found, clean up and bail out! */
-	list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
+	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
 		ret = drm_mm_scan_remove_block(&scan, &vma->node);
 		BUG_ON(ret);
-
-		INIT_LIST_HEAD(&vma->exec_list);
 	}
 
 	/* Can we unpin some objects such as idle hw contents,
@@ -180,19 +198,7 @@ search_again:
 		return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC;
 	}
 
-	/* Not everything in the GGTT is tracked via vma (otherwise we
-	 * could evict as required with minimal stalling) so we are forced
-	 * to idle the GPU and explicitly retire outstanding requests in
-	 * the hopes that we can then remove contexts and the like only
-	 * bound by their active reference.
-	 */
-	ret = i915_gem_switch_to_kernel_context(dev_priv);
-	if (ret)
-		return ret;
-
-	ret = i915_gem_wait_for_idle(dev_priv,
-				     I915_WAIT_INTERRUPTIBLE |
-				     I915_WAIT_LOCKED);
+	ret = ggtt_flush(dev_priv);
 	if (ret)
 		return ret;
 
@@ -205,21 +211,16 @@ found:
 	 * calling unbind (which may remove the active reference
 	 * of any of our objects, thus corrupting the list).
 	 */
-	list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
+	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
 		if (drm_mm_scan_remove_block(&scan, &vma->node))
 			__i915_vma_pin(vma);
 		else
-			list_del_init(&vma->exec_list);
+			list_del(&vma->evict_link);
 	}
 
 	/* Unbinding will emit any required flushes */
 	ret = 0;
-	while (!list_empty(&eviction_list)) {
-		vma = list_first_entry(&eviction_list,
-				       struct i915_vma,
-				       exec_list);
-
-		list_del_init(&vma->exec_list);
+	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
 		__i915_vma_unpin(vma);
 		if (ret == 0)
 			ret = i915_vma_unbind(vma);
@@ -315,7 +316,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 		}
 
 		/* Overlap of objects in the same batch? */
-		if (i915_vma_is_pinned(vma) || !list_empty(&vma->exec_list)) {
+		if (i915_vma_is_pinned(vma)) {
 			ret = -ENOSPC;
 			if (vma->exec_entry &&
 			    vma->exec_entry->flags & EXEC_OBJECT_PINNED)
@@ -332,11 +333,10 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 		 * reference) another in our eviction list.
 		 */
 		__i915_vma_pin(vma);
-		list_add(&vma->exec_list, &eviction_list);
+		list_add(&vma->evict_link, &eviction_list);
 	}
 
-	list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
-		list_del_init(&vma->exec_list);
+	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
 		__i915_vma_unpin(vma);
 		if (ret == 0)
 			ret = i915_vma_unbind(vma);
@@ -348,10 +348,8 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 /**
  * i915_gem_evict_vm - Evict all idle vmas from a vm
  * @vm: Address space to cleanse
- * @do_idle: Boolean directing whether to idle first.
  *
- * This function evicts all idles vmas from a vm. If all unpinned vmas should be
- * evicted the @do_idle needs to be set to true.
+ * This function evicts all vmas from a vm.
  *
  * This is used by the execbuf code as a last-ditch effort to defragment the
  * address space.
@@ -359,37 +357,50 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
  * To clarify: This is for freeing up virtual address space, not for freeing
  * memory in e.g. the shrinker.
  */
-int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
+int i915_gem_evict_vm(struct i915_address_space *vm)
 {
+	struct list_head *phases[] = {
+		&vm->inactive_list,
+		&vm->active_list,
+		NULL
+	}, **phase;
+	struct list_head eviction_list;
 	struct i915_vma *vma, *next;
 	int ret;
 
 	lockdep_assert_held(&vm->i915->drm.struct_mutex);
 	trace_i915_gem_evict_vm(vm);
 
-	if (do_idle) {
-		struct drm_i915_private *dev_priv = vm->i915;
-
-		if (i915_is_ggtt(vm)) {
-			ret = i915_gem_switch_to_kernel_context(dev_priv);
-			if (ret)
-				return ret;
-		}
-
-		ret = i915_gem_wait_for_idle(dev_priv,
-					     I915_WAIT_INTERRUPTIBLE |
-					     I915_WAIT_LOCKED);
+	/* Switch back to the default context in order to unpin
+	 * the existing context objects. However, such objects only
+	 * pin themselves inside the global GTT and performing the
+	 * switch otherwise is ineffective.
+	 */
+	if (i915_is_ggtt(vm)) {
+		ret = ggtt_flush(vm->i915);
 		if (ret)
 			return ret;
-
-		WARN_ON(!list_empty(&vm->active_list));
 	}
 
-	list_for_each_entry_safe(vma, next, &vm->inactive_list, vm_link)
-		if (!i915_vma_is_pinned(vma))
-			WARN_ON(i915_vma_unbind(vma));
+	INIT_LIST_HEAD(&eviction_list);
+	phase = phases;
+	do {
+		list_for_each_entry(vma, *phase, vm_link) {
+			if (i915_vma_is_pinned(vma))
+				continue;
 
-	return 0;
+			__i915_vma_pin(vma);
+			list_add(&vma->evict_link, &eviction_list);
+		}
+	} while (*++phase);
+
+	ret = 0;
+	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
+		__i915_vma_unpin(vma);
+		if (ret == 0)
+			ret = i915_vma_unbind(vma);
+	}
+	return ret;
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 9ad13eeed904..9337446f1068 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -40,143 +40,726 @@
 #include "intel_drv.h"
 #include "intel_frontbuffer.h"
 
-#define DBG_USE_CPU_RELOC 0 /* -1 force GTT relocs; 1 force CPU relocs */
+enum {
+	FORCE_CPU_RELOC = 1,
+	FORCE_GTT_RELOC,
+	FORCE_GPU_RELOC,
+#define DBG_FORCE_RELOC 0 /* choose one of the above! */
+};
+
+#define __EXEC_OBJECT_HAS_REF		BIT(31)
+#define __EXEC_OBJECT_HAS_PIN		BIT(30)
+#define __EXEC_OBJECT_HAS_FENCE		BIT(29)
+#define __EXEC_OBJECT_NEEDS_MAP		BIT(28)
+#define __EXEC_OBJECT_NEEDS_BIAS	BIT(27)
+#define __EXEC_OBJECT_INTERNAL_FLAGS	(~0u << 27) /* all of the above */
+#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
 
-#define  __EXEC_OBJECT_HAS_PIN		(1<<31)
-#define  __EXEC_OBJECT_HAS_FENCE	(1<<30)
-#define  __EXEC_OBJECT_NEEDS_MAP	(1<<29)
-#define  __EXEC_OBJECT_NEEDS_BIAS	(1<<28)
-#define  __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
+#define __EXEC_HAS_RELOC	BIT(31)
+#define __EXEC_VALIDATED	BIT(30)
+#define UPDATE			PIN_OFFSET_FIXED
 
 #define BATCH_OFFSET_BIAS (256*1024)
 
-struct i915_execbuffer_params {
-	struct drm_device               *dev;
-	struct drm_file                 *file;
-	struct i915_vma			*batch;
-	u32				dispatch_flags;
-	u32				args_batch_start_offset;
-	struct intel_engine_cs          *engine;
-	struct i915_gem_context         *ctx;
-	struct drm_i915_gem_request     *request;
-};
+#define __I915_EXEC_ILLEGAL_FLAGS \
+	(__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
+
+/**
+ * DOC: User command execution
+ *
+ * Userspace submits commands to be executed on the GPU as an instruction
+ * stream within a GEM object we call a batchbuffer. This instructions may
+ * refer to other GEM objects containing auxiliary state such as kernels,
+ * samplers, render targets and even secondary batchbuffers. Userspace does
+ * not know where in the GPU memory these objects reside and so before the
+ * batchbuffer is passed to the GPU for execution, those addresses in the
+ * batchbuffer and auxiliary objects are updated. This is known as relocation,
+ * or patching. To try and avoid having to relocate each object on the next
+ * execution, userspace is told the location of those objects in this pass,
+ * but this remains just a hint as the kernel may choose a new location for
+ * any object in the future.
+ *
+ * Processing an execbuf ioctl is conceptually split up into a few phases.
+ *
+ * 1. Validation - Ensure all the pointers, handles and flags are valid.
+ * 2. Reservation - Assign GPU address space for every object
+ * 3. Relocation - Update any addresses to point to the final locations
+ * 4. Serialisation - Order the request with respect to its dependencies
+ * 5. Construction - Construct a request to execute the batchbuffer
+ * 6. Submission (at some point in the future execution)
+ *
+ * Reserving resources for the execbuf is the most complicated phase. We
+ * neither want to have to migrate the object in the address space, nor do
+ * we want to have to update any relocations pointing to this object. Ideally,
+ * we want to leave the object where it is and for all the existing relocations
+ * to match. If the object is given a new address, or if userspace thinks the
+ * object is elsewhere, we have to parse all the relocation entries and update
+ * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
+ * all the target addresses in all of its objects match the value in the
+ * relocation entries and that they all match the presumed offsets given by the
+ * list of execbuffer objects. Using this knowledge, we know that if we haven't
+ * moved any buffers, all the relocation entries are valid and we can skip
+ * the update. (If userspace is wrong, the likely outcome is an impromptu GPU
+ * hang.) The requirement for using I915_EXEC_NO_RELOC are:
+ *
+ *      The addresses written in the objects must match the corresponding
+ *      reloc.presumed_offset which in turn must match the corresponding
+ *      execobject.offset.
+ *
+ *      Any render targets written to in the batch must be flagged with
+ *      EXEC_OBJECT_WRITE.
+ *
+ *      To avoid stalling, execobject.offset should match the current
+ *      address of that object within the active context.
+ *
+ * The reservation is done is multiple phases. First we try and keep any
+ * object already bound in its current location - so as long as meets the
+ * constraints imposed by the new execbuffer. Any object left unbound after the
+ * first pass is then fitted into any available idle space. If an object does
+ * not fit, all objects are removed from the reservation and the process rerun
+ * after sorting the objects into a priority order (more difficult to fit
+ * objects are tried first). Failing that, the entire VM is cleared and we try
+ * to fit the execbuf once last time before concluding that it simply will not
+ * fit.
+ *
+ * A small complication to all of this is that we allow userspace not only to
+ * specify an alignment and a size for the object in the address space, but
+ * we also allow userspace to specify the exact offset. This objects are
+ * simpler to place (the location is known a priori) all we have to do is make
+ * sure the space is available.
+ *
+ * Once all the objects are in place, patching up the buried pointers to point
+ * to the final locations is a fairly simple job of walking over the relocation
+ * entry arrays, looking up the right address and rewriting the value into
+ * the object. Simple! ... The relocation entries are stored in user memory
+ * and so to access them we have to copy them into a local buffer. That copy
+ * has to avoid taking any pagefaults as they may lead back to a GEM object
+ * requiring the struct_mutex (i.e. recursive deadlock). So once again we split
+ * the relocation into multiple passes. First we try to do everything within an
+ * atomic context (avoid the pagefaults) which requires that we never wait. If
+ * we detect that we may wait, or if we need to fault, then we have to fallback
+ * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
+ * bells yet?) Dropping the mutex means that we lose all the state we have
+ * built up so far for the execbuf and we must reset any global data. However,
+ * we do leave the objects pinned in their final locations - which is a
+ * potential issue for concurrent execbufs. Once we have left the mutex, we can
+ * allocate and copy all the relocation entries into a large array at our
+ * leisure, reacquire the mutex, reclaim all the objects and other state and
+ * then proceed to update any incorrect addresses with the objects.
+ *
+ * As we process the relocation entries, we maintain a record of whether the
+ * object is being written to. Using NORELOC, we expect userspace to provide
+ * this information instead. We also check whether we can skip the relocation
+ * by comparing the expected value inside the relocation entry with the target's
+ * final address. If they differ, we have to map the current object and rewrite
+ * the 4 or 8 byte pointer within.
+ *
+ * Serialising an execbuf is quite simple according to the rules of the GEM
+ * ABI. Execution within each context is ordered by the order of submission.
+ * Writes to any GEM object are in order of submission and are exclusive. Reads
+ * from a GEM object are unordered with respect to other reads, but ordered by
+ * writes. A write submitted after a read cannot occur before the read, and
+ * similarly any read submitted after a write cannot occur before the write.
+ * Writes are ordered between engines such that only one write occurs at any
+ * time (completing any reads beforehand) - using semaphores where available
+ * and CPU serialisation otherwise. Other GEM access obey the same rules, any
+ * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
+ * reads before starting, and any read (either using set-domain or pread) must
+ * flush all GPU writes before starting. (Note we only employ a barrier before,
+ * we currently rely on userspace not concurrently starting a new execution
+ * whilst reading or writing to an object. This may be an advantage or not
+ * depending on how much you trust userspace not to shoot themselves in the
+ * foot.) Serialisation may just result in the request being inserted into
+ * a DAG awaiting its turn, but most simple is to wait on the CPU until
+ * all dependencies are resolved.
+ *
+ * After all of that, is just a matter of closing the request and handing it to
+ * the hardware (well, leaving it in a queue to be executed). However, we also
+ * offer the ability for batchbuffers to be run with elevated privileges so
+ * that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
+ * Before any batch is given extra privileges we first must check that it
+ * contains no nefarious instructions, we check that each instruction is from
+ * our whitelist and all registers are also from an allowed list. We first
+ * copy the user's batchbuffer to a shadow (so that the user doesn't have
+ * access to it, either by the CPU or GPU as we scan it) and then parse each
+ * instruction. If everything is ok, we set a flag telling the hardware to run
+ * the batchbuffer in trusted mode, otherwise the ioctl is rejected.
+ */
+
+struct i915_execbuffer {
+	struct drm_i915_private *i915; /** i915 backpointer */
+	struct drm_file *file; /** per-file lookup tables and limits */
+	struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
+	struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
+
+	struct intel_engine_cs *engine; /** engine to queue the request to */
+	struct i915_gem_context *ctx; /** context for building the request */
+	struct i915_address_space *vm; /** GTT and vma for the request */
+
+	struct drm_i915_gem_request *request; /** our request to build */
+	struct i915_vma *batch; /** identity of the batch obj/vma */
+
+	/** actual size of execobj[] as we may extend it for the cmdparser */
+	unsigned int buffer_count;
+
+	/** list of vma not yet bound during reservation phase */
+	struct list_head unbound;
+
+	/** list of vma that have execobj.relocation_count */
+	struct list_head relocs;
 
-struct eb_vmas {
-	struct drm_i915_private *i915;
-	struct list_head vmas;
-	int and;
-	union {
-		struct i915_vma *lut[0];
-		struct hlist_head buckets[0];
-	};
+	/**
+	 * Track the most recently used object for relocations, as we
+	 * frequently have to perform multiple relocations within the same
+	 * obj/page
+	 */
+	struct reloc_cache {
+		struct drm_mm_node node; /** temporary GTT binding */
+		unsigned long vaddr; /** Current kmap address */
+		unsigned long page; /** Currently mapped page index */
+		unsigned int gen; /** Cached value of INTEL_GEN */
+		bool use_64bit_reloc : 1;
+		bool has_llc : 1;
+		bool has_fence : 1;
+		bool needs_unfenced : 1;
+
+		struct drm_i915_gem_request *rq;
+		u32 *rq_cmd;
+		unsigned int rq_size;
+	} reloc_cache;
+
+	u64 invalid_flags; /** Set of execobj.flags that are invalid */
+	u32 context_flags; /** Set of execobj.flags to insert from the ctx */
+
+	u32 batch_start_offset; /** Location within object of batch */
+	u32 batch_len; /** Length of batch within object */
+	u32 batch_flags; /** Flags composed for emit_bb_start() */
+
+	/**
+	 * Indicate either the size of the hastable used to resolve
+	 * relocation handles, or if negative that we are using a direct
+	 * index into the execobj[].
+	 */
+	int lut_size;
+	struct hlist_head *buckets; /** ht for relocation handles */
 };
 
-static struct eb_vmas *
-eb_create(struct drm_i915_private *i915,
-	  struct drm_i915_gem_execbuffer2 *args)
+/*
+ * As an alternative to creating a hashtable of handle-to-vma for a batch,
+ * we used the last available reserved field in the execobject[] and stash
+ * a link from the execobj to its vma.
+ */
+#define __exec_to_vma(ee) (ee)->rsvd2
+#define exec_to_vma(ee) u64_to_ptr(struct i915_vma, __exec_to_vma(ee))
+
+/*
+ * Used to convert any address to canonical form.
+ * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
+ * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
+ * addresses to be in a canonical form:
+ * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
+ * canonical form [63:48] == [47]."
+ */
+#define GEN8_HIGH_ADDRESS_BIT 47
+static inline u64 gen8_canonical_addr(u64 address)
 {
-	struct eb_vmas *eb = NULL;
-
-	if (args->flags & I915_EXEC_HANDLE_LUT) {
-		unsigned size = args->buffer_count;
-		size *= sizeof(struct i915_vma *);
-		size += sizeof(struct eb_vmas);
-		eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
-	}
-
-	if (eb == NULL) {
-		unsigned size = args->buffer_count;
-		unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
-		BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
-		while (count > 2*size)
-			count >>= 1;
-		eb = kzalloc(count*sizeof(struct hlist_head) +
-			     sizeof(struct eb_vmas),
-			     GFP_TEMPORARY);
-		if (eb == NULL)
-			return eb;
-
-		eb->and = count - 1;
-	} else
-		eb->and = -args->buffer_count;
+	return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
+}
 
-	eb->i915 = i915;
-	INIT_LIST_HEAD(&eb->vmas);
-	return eb;
+static inline u64 gen8_noncanonical_addr(u64 address)
+{
+	return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0);
 }
 
-static void
-eb_reset(struct eb_vmas *eb)
+static int eb_create(struct i915_execbuffer *eb)
 {
-	if (eb->and >= 0)
-		memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
+	if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
+		unsigned int size = 1 + ilog2(eb->buffer_count);
+
+		/*
+		 * Without a 1:1 association between relocation handles and
+		 * the execobject[] index, we instead create a hashtable.
+		 * We size it dynamically based on available memory, starting
+		 * first with 1:1 assocative hash and scaling back until
+		 * the allocation succeeds.
+		 *
+		 * Later on we use a positive lut_size to indicate we are
+		 * using this hashtable, and a negative value to indicate a
+		 * direct lookup.
+		 */
+		do {
+			eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
+					      GFP_TEMPORARY |
+					      __GFP_NORETRY |
+					      __GFP_NOWARN);
+			if (eb->buckets)
+				break;
+		} while (--size);
+
+		if (unlikely(!eb->buckets)) {
+			eb->buckets = kzalloc(sizeof(struct hlist_head),
+					      GFP_TEMPORARY);
+			if (unlikely(!eb->buckets))
+				return -ENOMEM;
+		}
+
+		eb->lut_size = size;
+	} else {
+		eb->lut_size = -eb->buffer_count;
+	}
+
+	return 0;
 }
 
-static struct i915_vma *
-eb_get_batch(struct eb_vmas *eb)
+static bool
+eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
+		 const struct i915_vma *vma)
 {
-	struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
+	if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+		return true;
+
+	if (vma->node.size < entry->pad_to_size)
+		return true;
+
+	if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
+		return true;
+
+	if (entry->flags & EXEC_OBJECT_PINNED &&
+	    vma->node.start != entry->offset)
+		return true;
+
+	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
+	    vma->node.start < BATCH_OFFSET_BIAS)
+		return true;
+
+	if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
+	    (vma->node.start + vma->node.size - 1) >> 32)
+		return true;
+
+	return false;
+}
+
+static inline void
+eb_pin_vma(struct i915_execbuffer *eb,
+	   struct drm_i915_gem_exec_object2 *entry,
+	   struct i915_vma *vma)
+{
+	u64 flags;
+
+	if (vma->node.size)
+		flags = vma->node.start;
+	else
+		flags = entry->offset & PIN_OFFSET_MASK;
+
+	flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
+	if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_GTT))
+		flags |= PIN_GLOBAL;
+
+	if (unlikely(i915_vma_pin(vma, 0, 0, flags)))
+		return;
+
+	if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+		if (unlikely(i915_vma_get_fence(vma))) {
+			i915_vma_unpin(vma);
+			return;
+		}
+
+		if (i915_vma_pin_fence(vma))
+			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+	}
+
+	entry->flags |= __EXEC_OBJECT_HAS_PIN;
+}
+
+static inline void
+__eb_unreserve_vma(struct i915_vma *vma,
+		   const struct drm_i915_gem_exec_object2 *entry)
+{
+	GEM_BUG_ON(!(entry->flags & __EXEC_OBJECT_HAS_PIN));
+
+	if (unlikely(entry->flags & __EXEC_OBJECT_HAS_FENCE))
+		i915_vma_unpin_fence(vma);
+
+	__i915_vma_unpin(vma);
+}
+
+static inline void
+eb_unreserve_vma(struct i915_vma *vma,
+		 struct drm_i915_gem_exec_object2 *entry)
+{
+	if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+		return;
+
+	__eb_unreserve_vma(vma, entry);
+	entry->flags &= ~__EXEC_OBJECT_RESERVED;
+}
+
+static int
+eb_validate_vma(struct i915_execbuffer *eb,
+		struct drm_i915_gem_exec_object2 *entry,
+		struct i915_vma *vma)
+{
+	if (unlikely(entry->flags & eb->invalid_flags))
+		return -EINVAL;
+
+	if (unlikely(entry->alignment && !is_power_of_2(entry->alignment)))
+		return -EINVAL;
 
 	/*
-	 * SNA is doing fancy tricks with compressing batch buffers, which leads
-	 * to negative relocation deltas. Usually that works out ok since the
-	 * relocate address is still positive, except when the batch is placed
-	 * very low in the GTT. Ensure this doesn't happen.
-	 *
-	 * Note that actual hangs have only been observed on gen7, but for
-	 * paranoia do it everywhere.
+	 * Offset can be used as input (EXEC_OBJECT_PINNED), reject
+	 * any non-page-aligned or non-canonical addresses.
 	 */
-	if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
-		vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
+	if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
+		     entry->offset != gen8_canonical_addr(entry->offset & PAGE_MASK)))
+		return -EINVAL;
 
-	return vma;
+	/* pad_to_size was once a reserved field, so sanitize it */
+	if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
+		if (unlikely(offset_in_page(entry->pad_to_size)))
+			return -EINVAL;
+	} else {
+		entry->pad_to_size = 0;
+	}
+
+	if (unlikely(vma->exec_entry)) {
+		DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
+			  entry->handle, (int)(entry - eb->exec));
+		return -EINVAL;
+	}
+
+	/*
+	 * From drm_mm perspective address space is continuous,
+	 * so from this point we're always using non-canonical
+	 * form internally.
+	 */
+	entry->offset = gen8_noncanonical_addr(entry->offset);
+
+	return 0;
 }
 
 static int
-eb_lookup_vmas(struct eb_vmas *eb,
-	       struct drm_i915_gem_exec_object2 *exec,
-	       const struct drm_i915_gem_execbuffer2 *args,
-	       struct i915_address_space *vm,
-	       struct drm_file *file)
+eb_add_vma(struct i915_execbuffer *eb,
+	   struct drm_i915_gem_exec_object2 *entry,
+	   struct i915_vma *vma)
 {
-	struct drm_i915_gem_object *obj;
-	struct list_head objects;
-	int i, ret;
+	int err;
 
-	INIT_LIST_HEAD(&objects);
-	spin_lock(&file->table_lock);
-	/* Grab a reference to the object and release the lock so we can lookup
-	 * or create the VMA without using GFP_ATOMIC */
-	for (i = 0; i < args->buffer_count; i++) {
-		obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
-		if (obj == NULL) {
-			spin_unlock(&file->table_lock);
-			DRM_DEBUG("Invalid object handle %d at index %d\n",
-				   exec[i].handle, i);
-			ret = -ENOENT;
-			goto err;
+	GEM_BUG_ON(i915_vma_is_closed(vma));
+
+	if (!(eb->args->flags & __EXEC_VALIDATED)) {
+		err = eb_validate_vma(eb, entry, vma);
+		if (unlikely(err))
+			return err;
+	}
+
+	if (eb->lut_size >= 0) {
+		vma->exec_handle = entry->handle;
+		hlist_add_head(&vma->exec_node,
+			       &eb->buckets[hash_32(entry->handle,
+						    eb->lut_size)]);
+	}
+
+	if (entry->relocation_count)
+		list_add_tail(&vma->reloc_link, &eb->relocs);
+
+	if (!eb->reloc_cache.has_fence) {
+		entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
+	} else {
+		if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
+		     eb->reloc_cache.needs_unfenced) &&
+		    i915_gem_object_is_tiled(vma->obj))
+			entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
+	}
+
+	if (!(entry->flags & EXEC_OBJECT_PINNED))
+		entry->flags |= eb->context_flags;
+
+	/*
+	 * Stash a pointer from the vma to execobj, so we can query its flags,
+	 * size, alignment etc as provided by the user. Also we stash a pointer
+	 * to the vma inside the execobj so that we can use a direct lookup
+	 * to find the right target VMA when doing relocations.
+	 */
+	vma->exec_entry = entry;
+	__exec_to_vma(entry) = (uintptr_t)vma;
+
+	err = 0;
+	eb_pin_vma(eb, entry, vma);
+	if (eb_vma_misplaced(entry, vma)) {
+		eb_unreserve_vma(vma, entry);
+
+		list_add_tail(&vma->exec_link, &eb->unbound);
+		if (drm_mm_node_allocated(&vma->node))
+			err = i915_vma_unbind(vma);
+	} else {
+		if (entry->offset != vma->node.start) {
+			entry->offset = vma->node.start | UPDATE;
+			eb->args->flags |= __EXEC_HAS_RELOC;
 		}
+	}
+	return err;
+}
+
+static inline int use_cpu_reloc(const struct reloc_cache *cache,
+				const struct drm_i915_gem_object *obj)
+{
+	if (!i915_gem_object_has_struct_page(obj))
+		return false;
+
+	if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
+		return true;
+
+	if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
+		return false;
+
+	return (cache->has_llc ||
+		obj->cache_dirty ||
+		obj->cache_level != I915_CACHE_NONE);
+}
+
+static int eb_reserve_vma(const struct i915_execbuffer *eb,
+			  struct i915_vma *vma)
+{
+	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+	u64 flags;
+	int err;
+
+	flags = PIN_USER | PIN_NONBLOCK;
+	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
+		flags |= PIN_GLOBAL;
+
+	/*
+	 * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
+	 * limit address to the first 4GBs for unflagged objects.
+	 */
+	if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
+		flags |= PIN_ZONE_4G;
 
-		if (!list_empty(&obj->obj_exec_link)) {
-			spin_unlock(&file->table_lock);
-			DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
-				   obj, exec[i].handle, i);
-			ret = -EINVAL;
+	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+		flags |= PIN_MAPPABLE;
+
+	if (entry->flags & EXEC_OBJECT_PINNED) {
+		flags |= entry->offset | PIN_OFFSET_FIXED;
+		flags &= ~PIN_NONBLOCK; /* force overlapping PINNED checks */
+	} else if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) {
+		flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+	}
+
+	err = i915_vma_pin(vma, entry->pad_to_size, entry->alignment, flags);
+	if (err)
+		return err;
+
+	if (entry->offset != vma->node.start) {
+		entry->offset = vma->node.start | UPDATE;
+		eb->args->flags |= __EXEC_HAS_RELOC;
+	}
+
+	entry->flags |= __EXEC_OBJECT_HAS_PIN;
+	GEM_BUG_ON(eb_vma_misplaced(entry, vma));
+
+	if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+		err = i915_vma_get_fence(vma);
+		if (unlikely(err)) {
+			i915_vma_unpin(vma);
+			return err;
+		}
+
+		if (i915_vma_pin_fence(vma))
+			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+	}
+
+	return 0;
+}
+
+static int eb_reserve(struct i915_execbuffer *eb)
+{
+	const unsigned int count = eb->buffer_count;
+	struct list_head last;
+	struct i915_vma *vma;
+	unsigned int i, pass;
+	int err;
+
+	/*
+	 * Attempt to pin all of the buffers into the GTT.
+	 * This is done in 3 phases:
+	 *
+	 * 1a. Unbind all objects that do not match the GTT constraints for
+	 *     the execbuffer (fenceable, mappable, alignment etc).
+	 * 1b. Increment pin count for already bound objects.
+	 * 2.  Bind new objects.
+	 * 3.  Decrement pin count.
+	 *
+	 * This avoid unnecessary unbinding of later objects in order to make
+	 * room for the earlier objects *unless* we need to defragment.
+	 */
+
+	pass = 0;
+	err = 0;
+	do {
+		list_for_each_entry(vma, &eb->unbound, exec_link) {
+			err = eb_reserve_vma(eb, vma);
+			if (err)
+				break;
+		}
+		if (err != -ENOSPC)
+			return err;
+
+		/* Resort *all* the objects into priority order */
+		INIT_LIST_HEAD(&eb->unbound);
+		INIT_LIST_HEAD(&last);
+		for (i = 0; i < count; i++) {
+			struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+
+			if (entry->flags & EXEC_OBJECT_PINNED &&
+			    entry->flags & __EXEC_OBJECT_HAS_PIN)
+				continue;
+
+			vma = exec_to_vma(entry);
+			eb_unreserve_vma(vma, entry);
+
+			if (entry->flags & EXEC_OBJECT_PINNED)
+				list_add(&vma->exec_link, &eb->unbound);
+			else if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+				list_add_tail(&vma->exec_link, &eb->unbound);
+			else
+				list_add_tail(&vma->exec_link, &last);
+		}
+		list_splice_tail(&last, &eb->unbound);
+
+		switch (pass++) {
+		case 0:
+			break;
+
+		case 1:
+			/* Too fragmented, unbind everything and retry */
+			err = i915_gem_evict_vm(eb->vm);
+			if (err)
+				return err;
+			break;
+
+		default:
+			return -ENOSPC;
+		}
+	} while (1);
+}
+
+static inline struct hlist_head *
+ht_head(const  struct i915_gem_context_vma_lut *lut, u32 handle)
+{
+	return &lut->ht[hash_32(handle, lut->ht_bits)];
+}
+
+static inline bool
+ht_needs_resize(const struct i915_gem_context_vma_lut *lut)
+{
+	return (4*lut->ht_count > 3*lut->ht_size ||
+		4*lut->ht_count + 1 < lut->ht_size);
+}
+
+static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
+{
+	if (eb->args->flags & I915_EXEC_BATCH_FIRST)
+		return 0;
+	else
+		return eb->buffer_count - 1;
+}
+
+static int eb_select_context(struct i915_execbuffer *eb)
+{
+	struct i915_gem_context *ctx;
+
+	ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
+	if (unlikely(IS_ERR(ctx)))
+		return PTR_ERR(ctx);
+
+	if (unlikely(i915_gem_context_is_banned(ctx))) {
+		DRM_DEBUG("Context %u tried to submit while banned\n",
+			  ctx->user_handle);
+		return -EIO;
+	}
+
+	eb->ctx = i915_gem_context_get(ctx);
+	eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
+
+	eb->context_flags = 0;
+	if (ctx->flags & CONTEXT_NO_ZEROMAP)
+		eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
+
+	return 0;
+}
+
+static int eb_lookup_vmas(struct i915_execbuffer *eb)
+{
+#define INTERMEDIATE BIT(0)
+	const unsigned int count = eb->buffer_count;
+	struct i915_gem_context_vma_lut *lut = &eb->ctx->vma_lut;
+	struct i915_vma *vma;
+	struct idr *idr;
+	unsigned int i;
+	int slow_pass = -1;
+	int err;
+
+	INIT_LIST_HEAD(&eb->relocs);
+	INIT_LIST_HEAD(&eb->unbound);
+
+	if (unlikely(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS))
+		flush_work(&lut->resize);
+	GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
+
+	for (i = 0; i < count; i++) {
+		__exec_to_vma(&eb->exec[i]) = 0;
+
+		hlist_for_each_entry(vma,
+				     ht_head(lut, eb->exec[i].handle),
+				     ctx_node) {
+			if (vma->ctx_handle != eb->exec[i].handle)
+				continue;
+
+			err = eb_add_vma(eb, &eb->exec[i], vma);
+			if (unlikely(err))
+				return err;
+
+			goto next_vma;
+		}
+
+		if (slow_pass < 0)
+			slow_pass = i;
+next_vma: ;
+	}
+
+	if (slow_pass < 0)
+		goto out;
+
+	spin_lock(&eb->file->table_lock);
+	/*
+	 * Grab a reference to the object and release the lock so we can lookup
+	 * or create the VMA without using GFP_ATOMIC
+	 */
+	idr = &eb->file->object_idr;
+	for (i = slow_pass; i < count; i++) {
+		struct drm_i915_gem_object *obj;
+
+		if (__exec_to_vma(&eb->exec[i]))
+			continue;
+
+		obj = to_intel_bo(idr_find(idr, eb->exec[i].handle));
+		if (unlikely(!obj)) {
+			spin_unlock(&eb->file->table_lock);
+			DRM_DEBUG("Invalid object handle %d at index %d\n",
+				  eb->exec[i].handle, i);
+			err = -ENOENT;
 			goto err;
 		}
 
-		i915_gem_object_get(obj);
-		list_add_tail(&obj->obj_exec_link, &objects);
+		__exec_to_vma(&eb->exec[i]) = INTERMEDIATE | (uintptr_t)obj;
 	}
-	spin_unlock(&file->table_lock);
+	spin_unlock(&eb->file->table_lock);
 
-	i = 0;
-	while (!list_empty(&objects)) {
-		struct i915_vma *vma;
+	for (i = slow_pass; i < count; i++) {
+		struct drm_i915_gem_object *obj;
 
-		obj = list_first_entry(&objects,
-				       struct drm_i915_gem_object,
-				       obj_exec_link);
+		if (!(__exec_to_vma(&eb->exec[i]) & INTERMEDIATE))
+			continue;
 
 		/*
 		 * NOTE: We can leak any vmas created here when something fails
@@ -186,59 +769,93 @@ eb_lookup_vmas(struct eb_vmas *eb,
 		 * from the (obj, vm) we don't run the risk of creating
 		 * duplicated vmas for the same vm.
 		 */
-		vma = i915_vma_instance(obj, vm, NULL);
+		obj = u64_to_ptr(typeof(*obj),
+				 __exec_to_vma(&eb->exec[i]) & ~INTERMEDIATE);
+		vma = i915_vma_instance(obj, eb->vm, NULL);
 		if (unlikely(IS_ERR(vma))) {
 			DRM_DEBUG("Failed to lookup VMA\n");
-			ret = PTR_ERR(vma);
+			err = PTR_ERR(vma);
 			goto err;
 		}
 
-		/* Transfer ownership from the objects list to the vmas list. */
-		list_add_tail(&vma->exec_list, &eb->vmas);
-		list_del_init(&obj->obj_exec_link);
+		/* First come, first served */
+		if (!vma->ctx) {
+			vma->ctx = eb->ctx;
+			vma->ctx_handle = eb->exec[i].handle;
+			hlist_add_head(&vma->ctx_node,
+				       ht_head(lut, eb->exec[i].handle));
+			lut->ht_count++;
+			lut->ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
+			if (i915_vma_is_ggtt(vma)) {
+				GEM_BUG_ON(obj->vma_hashed);
+				obj->vma_hashed = vma;
+			}
 
-		vma->exec_entry = &exec[i];
-		if (eb->and < 0) {
-			eb->lut[i] = vma;
-		} else {
-			uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
-			vma->exec_handle = handle;
-			hlist_add_head(&vma->exec_node,
-				       &eb->buckets[handle & eb->and]);
+			i915_vma_get(vma);
 		}
-		++i;
-	}
 
-	return 0;
+		err = eb_add_vma(eb, &eb->exec[i], vma);
+		if (unlikely(err))
+			goto err;
 
+		/* Only after we validated the user didn't use our bits */
+		if (vma->ctx != eb->ctx) {
+			i915_vma_get(vma);
+			eb->exec[i].flags |= __EXEC_OBJECT_HAS_REF;
+		}
+	}
 
-err:
-	while (!list_empty(&objects)) {
-		obj = list_first_entry(&objects,
-				       struct drm_i915_gem_object,
-				       obj_exec_link);
-		list_del_init(&obj->obj_exec_link);
-		i915_gem_object_put(obj);
+	if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
+		if (ht_needs_resize(lut))
+			queue_work(system_highpri_wq, &lut->resize);
+		else
+			lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
 	}
+
+out:
+	/* take note of the batch buffer before we might reorder the lists */
+	i = eb_batch_index(eb);
+	eb->batch = exec_to_vma(&eb->exec[i]);
+
 	/*
-	 * Objects already transfered to the vmas list will be unreferenced by
-	 * eb_destroy.
+	 * SNA is doing fancy tricks with compressing batch buffers, which leads
+	 * to negative relocation deltas. Usually that works out ok since the
+	 * relocate address is still positive, except when the batch is placed
+	 * very low in the GTT. Ensure this doesn't happen.
+	 *
+	 * Note that actual hangs have only been observed on gen7, but for
+	 * paranoia do it everywhere.
 	 */
+	if (!(eb->exec[i].flags & EXEC_OBJECT_PINNED))
+		eb->exec[i].flags |= __EXEC_OBJECT_NEEDS_BIAS;
+	if (eb->reloc_cache.has_fence)
+		eb->exec[i].flags |= EXEC_OBJECT_NEEDS_FENCE;
 
-	return ret;
+	eb->args->flags |= __EXEC_VALIDATED;
+	return eb_reserve(eb);
+
+err:
+	for (i = slow_pass; i < count; i++) {
+		if (__exec_to_vma(&eb->exec[i]) & INTERMEDIATE)
+			__exec_to_vma(&eb->exec[i]) = 0;
+	}
+	lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+	return err;
+#undef INTERMEDIATE
 }
 
-static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
+static struct i915_vma *
+eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
 {
-	if (eb->and < 0) {
-		if (handle >= -eb->and)
+	if (eb->lut_size < 0) {
+		if (handle >= -eb->lut_size)
 			return NULL;
-		return eb->lut[handle];
+		return exec_to_vma(&eb->exec[handle]);
 	} else {
 		struct hlist_head *head;
 		struct i915_vma *vma;
 
-		head = &eb->buckets[handle & eb->and];
+		head = &eb->buckets[hash_32(handle, eb->lut_size)];
 		hlist_for_each_entry(vma, head, exec_node) {
 			if (vma->exec_handle == handle)
 				return vma;
@@ -247,96 +864,70 @@ static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
 	}
 }
 
-static void
-i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
+static void eb_release_vmas(const struct i915_execbuffer *eb)
 {
-	struct drm_i915_gem_exec_object2 *entry;
-
-	if (!drm_mm_node_allocated(&vma->node))
-		return;
+	const unsigned int count = eb->buffer_count;
+	unsigned int i;
 
-	entry = vma->exec_entry;
+	for (i = 0; i < count; i++) {
+		struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+		struct i915_vma *vma = exec_to_vma(entry);
 
-	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
-		i915_vma_unpin_fence(vma);
+		if (!vma)
+			continue;
 
-	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
-		__i915_vma_unpin(vma);
+		GEM_BUG_ON(vma->exec_entry != entry);
+		vma->exec_entry = NULL;
+		__exec_to_vma(entry) = 0;
 
-	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
-}
+		if (entry->flags & __EXEC_OBJECT_HAS_PIN)
+			__eb_unreserve_vma(vma, entry);
 
-static void eb_destroy(struct eb_vmas *eb)
-{
-	while (!list_empty(&eb->vmas)) {
-		struct i915_vma *vma;
+		if (entry->flags & __EXEC_OBJECT_HAS_REF)
+			i915_vma_put(vma);
 
-		vma = list_first_entry(&eb->vmas,
-				       struct i915_vma,
-				       exec_list);
-		list_del_init(&vma->exec_list);
-		i915_gem_execbuffer_unreserve_vma(vma);
-		vma->exec_entry = NULL;
-		i915_vma_put(vma);
+		entry->flags &=
+			~(__EXEC_OBJECT_RESERVED | __EXEC_OBJECT_HAS_REF);
 	}
-	kfree(eb);
 }
 
-static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
+static void eb_reset_vmas(const struct i915_execbuffer *eb)
 {
-	if (!i915_gem_object_has_struct_page(obj))
-		return false;
-
-	if (DBG_USE_CPU_RELOC)
-		return DBG_USE_CPU_RELOC > 0;
-
-	return (HAS_LLC(to_i915(obj->base.dev)) ||
-		obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
-		obj->cache_level != I915_CACHE_NONE);
+	eb_release_vmas(eb);
+	if (eb->lut_size >= 0)
+		memset(eb->buckets, 0,
+		       sizeof(struct hlist_head) << eb->lut_size);
 }
 
-/* Used to convert any address to canonical form.
- * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
- * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
- * addresses to be in a canonical form:
- * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
- * canonical form [63:48] == [47]."
- */
-#define GEN8_HIGH_ADDRESS_BIT 47
-static inline uint64_t gen8_canonical_addr(uint64_t address)
+static void eb_destroy(const struct i915_execbuffer *eb)
 {
-	return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
-}
+	GEM_BUG_ON(eb->reloc_cache.rq);
 
-static inline uint64_t gen8_noncanonical_addr(uint64_t address)
-{
-	return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
+	if (eb->lut_size >= 0)
+		kfree(eb->buckets);
 }
 
-static inline uint64_t
+static inline u64
 relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
-		  uint64_t target_offset)
+		  const struct i915_vma *target)
 {
-	return gen8_canonical_addr((int)reloc->delta + target_offset);
+	return gen8_canonical_addr((int)reloc->delta + target->node.start);
 }
 
-struct reloc_cache {
-	struct drm_i915_private *i915;
-	struct drm_mm_node node;
-	unsigned long vaddr;
-	unsigned int page;
-	bool use_64bit_reloc;
-};
-
 static void reloc_cache_init(struct reloc_cache *cache,
 			     struct drm_i915_private *i915)
 {
 	cache->page = -1;
 	cache->vaddr = 0;
-	cache->i915 = i915;
 	/* Must be a variable in the struct to allow GCC to unroll. */
+	cache->gen = INTEL_GEN(i915);
+	cache->has_llc = HAS_LLC(i915);
 	cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
+	cache->has_fence = cache->gen < 4;
+	cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
 	cache->node.allocated = false;
+	cache->rq = NULL;
+	cache->rq_size = 0;
 }
 
 static inline void *unmask_page(unsigned long p)
@@ -351,10 +942,31 @@ static inline unsigned int unmask_flags(unsigned long p)
 
 #define KMAP 0x4 /* after CLFLUSH_FLAGS */
 
-static void reloc_cache_fini(struct reloc_cache *cache)
+static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
+{
+	struct drm_i915_private *i915 =
+		container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
+	return &i915->ggtt;
+}
+
+static void reloc_gpu_flush(struct reloc_cache *cache)
+{
+	GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32));
+	cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
+	i915_gem_object_unpin_map(cache->rq->batch->obj);
+	i915_gem_chipset_flush(cache->rq->i915);
+
+	__i915_add_request(cache->rq, true);
+	cache->rq = NULL;
+}
+
+static void reloc_cache_reset(struct reloc_cache *cache)
 {
 	void *vaddr;
 
+	if (cache->rq)
+		reloc_gpu_flush(cache);
+
 	if (!cache->vaddr)
 		return;
 
@@ -369,7 +981,7 @@ static void reloc_cache_fini(struct reloc_cache *cache)
 		wmb();
 		io_mapping_unmap_atomic((void __iomem *)vaddr);
 		if (cache->node.allocated) {
-			struct i915_ggtt *ggtt = &cache->i915->ggtt;
+			struct i915_ggtt *ggtt = cache_to_ggtt(cache);
 
 			ggtt->base.clear_range(&ggtt->base,
 					       cache->node.start,
@@ -379,11 +991,14 @@ static void reloc_cache_fini(struct reloc_cache *cache)
 			i915_vma_unpin((struct i915_vma *)cache->node.mm);
 		}
 	}
+
+	cache->vaddr = 0;
+	cache->page = -1;
 }
 
 static void *reloc_kmap(struct drm_i915_gem_object *obj,
 			struct reloc_cache *cache,
-			int page)
+			unsigned long page)
 {
 	void *vaddr;
 
@@ -391,11 +1006,11 @@ static void *reloc_kmap(struct drm_i915_gem_object *obj,
 		kunmap_atomic(unmask_page(cache->vaddr));
 	} else {
 		unsigned int flushes;
-		int ret;
+		int err;
 
-		ret = i915_gem_obj_prepare_shmem_write(obj, &flushes);
-		if (ret)
-			return ERR_PTR(ret);
+		err = i915_gem_obj_prepare_shmem_write(obj, &flushes);
+		if (err)
+			return ERR_PTR(err);
 
 		BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
 		BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
@@ -415,9 +1030,9 @@ static void *reloc_kmap(struct drm_i915_gem_object *obj,
 
 static void *reloc_iomap(struct drm_i915_gem_object *obj,
 			 struct reloc_cache *cache,
-			 int page)
+			 unsigned long page)
 {
-	struct i915_ggtt *ggtt = &cache->i915->ggtt;
+	struct i915_ggtt *ggtt = cache_to_ggtt(cache);
 	unsigned long offset;
 	void *vaddr;
 
@@ -425,31 +1040,31 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 		io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
 	} else {
 		struct i915_vma *vma;
-		int ret;
+		int err;
 
-		if (use_cpu_reloc(obj))
+		if (use_cpu_reloc(cache, obj))
 			return NULL;
 
-		ret = i915_gem_object_set_to_gtt_domain(obj, true);
-		if (ret)
-			return ERR_PTR(ret);
+		err = i915_gem_object_set_to_gtt_domain(obj, true);
+		if (err)
+			return ERR_PTR(err);
 
 		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 					       PIN_MAPPABLE | PIN_NONBLOCK);
 		if (IS_ERR(vma)) {
 			memset(&cache->node, 0, sizeof(cache->node));
-			ret = drm_mm_insert_node_in_range
+			err = drm_mm_insert_node_in_range
 				(&ggtt->base.mm, &cache->node,
 				 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
 				 0, ggtt->mappable_end,
 				 DRM_MM_INSERT_LOW);
-			if (ret) /* no inactive aperture space, use cpu reloc */
+			if (err) /* no inactive aperture space, use cpu reloc */
 				return NULL;
 		} else {
-			ret = i915_vma_put_fence(vma);
-			if (ret) {
+			err = i915_vma_put_fence(vma);
+			if (err) {
 				i915_vma_unpin(vma);
-				return ERR_PTR(ret);
+				return ERR_PTR(err);
 			}
 
 			cache->node.start = vma->node.start;
@@ -467,7 +1082,8 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 		offset += page << PAGE_SHIFT;
 	}
 
-	vaddr = (void __force *) io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
+	vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->mappable,
+							 offset);
 	cache->page = page;
 	cache->vaddr = (unsigned long)vaddr;
 
@@ -476,7 +1092,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 
 static void *reloc_vaddr(struct drm_i915_gem_object *obj,
 			 struct reloc_cache *cache,
-			 int page)
+			 unsigned long page)
 {
 	void *vaddr;
 
@@ -503,7 +1119,8 @@ static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
 
 		*addr = value;
 
-		/* Writes to the same cacheline are serialised by the CPU
+		/*
+		 * Writes to the same cacheline are serialised by the CPU
 		 * (including clflush). On the write path, we only require
 		 * that it hits memory in an orderly fashion and place
 		 * mb barriers at the start and end of the relocation phase
@@ -515,25 +1132,201 @@ static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
 		*addr = value;
 }
 
-static int
-relocate_entry(struct drm_i915_gem_object *obj,
+static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
+			     struct i915_vma *vma,
+			     unsigned int len)
+{
+	struct reloc_cache *cache = &eb->reloc_cache;
+	struct drm_i915_gem_object *obj;
+	struct drm_i915_gem_request *rq;
+	struct i915_vma *batch;
+	u32 *cmd;
+	int err;
+
+	GEM_BUG_ON(vma->obj->base.write_domain & I915_GEM_DOMAIN_CPU);
+
+	obj = i915_gem_batch_pool_get(&eb->engine->batch_pool, PAGE_SIZE);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	cmd = i915_gem_object_pin_map(obj,
+				      cache->has_llc ? I915_MAP_WB : I915_MAP_WC);
+	i915_gem_object_unpin_pages(obj);
+	if (IS_ERR(cmd))
+		return PTR_ERR(cmd);
+
+	err = i915_gem_object_set_to_wc_domain(obj, false);
+	if (err)
+		goto err_unmap;
+
+	batch = i915_vma_instance(obj, vma->vm, NULL);
+	if (IS_ERR(batch)) {
+		err = PTR_ERR(batch);
+		goto err_unmap;
+	}
+
+	err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
+	if (err)
+		goto err_unmap;
+
+	rq = i915_gem_request_alloc(eb->engine, eb->ctx);
+	if (IS_ERR(rq)) {
+		err = PTR_ERR(rq);
+		goto err_unpin;
+	}
+
+	err = i915_gem_request_await_object(rq, vma->obj, true);
+	if (err)
+		goto err_request;
+
+	err = eb->engine->emit_flush(rq, EMIT_INVALIDATE);
+	if (err)
+		goto err_request;
+
+	err = i915_switch_context(rq);
+	if (err)
+		goto err_request;
+
+	err = eb->engine->emit_bb_start(rq,
+					batch->node.start, PAGE_SIZE,
+					cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
+	if (err)
+		goto err_request;
+
+	GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true));
+	i915_vma_move_to_active(batch, rq, 0);
+	reservation_object_lock(batch->resv, NULL);
+	reservation_object_add_excl_fence(batch->resv, &rq->fence);
+	reservation_object_unlock(batch->resv);
+	i915_vma_unpin(batch);
+
+	i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	reservation_object_lock(vma->resv, NULL);
+	reservation_object_add_excl_fence(vma->resv, &rq->fence);
+	reservation_object_unlock(vma->resv);
+
+	rq->batch = batch;
+
+	cache->rq = rq;
+	cache->rq_cmd = cmd;
+	cache->rq_size = 0;
+
+	/* Return with batch mapping (cmd) still pinned */
+	return 0;
+
+err_request:
+	i915_add_request(rq);
+err_unpin:
+	i915_vma_unpin(batch);
+err_unmap:
+	i915_gem_object_unpin_map(obj);
+	return err;
+}
+
+static u32 *reloc_gpu(struct i915_execbuffer *eb,
+		      struct i915_vma *vma,
+		      unsigned int len)
+{
+	struct reloc_cache *cache = &eb->reloc_cache;
+	u32 *cmd;
+
+	if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))
+		reloc_gpu_flush(cache);
+
+	if (unlikely(!cache->rq)) {
+		int err;
+
+		err = __reloc_gpu_alloc(eb, vma, len);
+		if (unlikely(err))
+			return ERR_PTR(err);
+	}
+
+	cmd = cache->rq_cmd + cache->rq_size;
+	cache->rq_size += len;
+
+	return cmd;
+}
+
+static u64
+relocate_entry(struct i915_vma *vma,
 	       const struct drm_i915_gem_relocation_entry *reloc,
-	       struct reloc_cache *cache,
-	       u64 target_offset)
+	       struct i915_execbuffer *eb,
+	       const struct i915_vma *target)
 {
 	u64 offset = reloc->offset;
-	bool wide = cache->use_64bit_reloc;
+	u64 target_offset = relocation_target(reloc, target);
+	bool wide = eb->reloc_cache.use_64bit_reloc;
 	void *vaddr;
 
-	target_offset = relocation_target(reloc, target_offset);
+	if (!eb->reloc_cache.vaddr &&
+	    (DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
+	     !reservation_object_test_signaled_rcu(vma->resv, true))) {
+		const unsigned int gen = eb->reloc_cache.gen;
+		unsigned int len;
+		u32 *batch;
+		u64 addr;
+
+		if (wide)
+			len = offset & 7 ? 8 : 5;
+		else if (gen >= 4)
+			len = 4;
+		else if (gen >= 3)
+			len = 3;
+		else /* On gen2 MI_STORE_DWORD_IMM uses a physical address */
+			goto repeat;
+
+		batch = reloc_gpu(eb, vma, len);
+		if (IS_ERR(batch))
+			goto repeat;
+
+		addr = gen8_canonical_addr(vma->node.start + offset);
+		if (wide) {
+			if (offset & 7) {
+				*batch++ = MI_STORE_DWORD_IMM_GEN4;
+				*batch++ = lower_32_bits(addr);
+				*batch++ = upper_32_bits(addr);
+				*batch++ = lower_32_bits(target_offset);
+
+				addr = gen8_canonical_addr(addr + 4);
+
+				*batch++ = MI_STORE_DWORD_IMM_GEN4;
+				*batch++ = lower_32_bits(addr);
+				*batch++ = upper_32_bits(addr);
+				*batch++ = upper_32_bits(target_offset);
+			} else {
+				*batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1;
+				*batch++ = lower_32_bits(addr);
+				*batch++ = upper_32_bits(addr);
+				*batch++ = lower_32_bits(target_offset);
+				*batch++ = upper_32_bits(target_offset);
+			}
+		} else if (gen >= 6) {
+			*batch++ = MI_STORE_DWORD_IMM_GEN4;
+			*batch++ = 0;
+			*batch++ = addr;
+			*batch++ = target_offset;
+		} else if (gen >= 4) {
+			*batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+			*batch++ = 0;
+			*batch++ = addr;
+			*batch++ = target_offset;
+		} else {
+			*batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
+			*batch++ = addr;
+			*batch++ = target_offset;
+		}
+
+		goto out;
+	}
+
 repeat:
-	vaddr = reloc_vaddr(obj, cache, offset >> PAGE_SHIFT);
+	vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT);
 	if (IS_ERR(vaddr))
 		return PTR_ERR(vaddr);
 
 	clflush_write32(vaddr + offset_in_page(offset),
 			lower_32_bits(target_offset),
-			cache->vaddr);
+			eb->reloc_cache.vaddr);
 
 	if (wide) {
 		offset += sizeof(u32);
@@ -542,49 +1335,29 @@ repeat:
 		goto repeat;
 	}
 
-	return 0;
+out:
+	return target->node.start | UPDATE;
 }
 
-static int
-i915_gem_execbuffer_relocate_entry(struct i915_vma *vma,
-				   struct eb_vmas *eb,
-				   struct drm_i915_gem_relocation_entry *reloc,
-				   struct reloc_cache *cache)
+static u64
+eb_relocate_entry(struct i915_execbuffer *eb,
+		  struct i915_vma *vma,
+		  const struct drm_i915_gem_relocation_entry *reloc)
 {
-	struct drm_i915_gem_object *obj = vma->obj;
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	struct drm_gem_object *target_obj;
-	struct drm_i915_gem_object *target_i915_obj;
-	struct i915_vma *target_vma;
-	uint64_t target_offset;
-	int ret;
+	struct i915_vma *target;
+	int err;
 
 	/* we've already hold a reference to all valid objects */
-	target_vma = eb_get_vma(eb, reloc->target_handle);
-	if (unlikely(target_vma == NULL))
+	target = eb_get_vma(eb, reloc->target_handle);
+	if (unlikely(!target))
 		return -ENOENT;
-	target_i915_obj = target_vma->obj;
-	target_obj = &target_vma->obj->base;
-
-	target_offset = gen8_canonical_addr(target_vma->node.start);
-
-	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
-	 * pipe_control writes because the gpu doesn't properly redirect them
-	 * through the ppgtt for non_secure batchbuffers. */
-	if (unlikely(IS_GEN6(dev_priv) &&
-	    reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
-		ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
-				    PIN_GLOBAL);
-		if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
-			return ret;
-	}
 
 	/* Validate that the target is in a valid r/w GPU domain */
 	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
 		DRM_DEBUG("reloc with multiple write domains: "
-			  "obj %p target %d offset %d "
+			  "target %d offset %d "
 			  "read %08x write %08x",
-			  obj, reloc->target_handle,
+			  reloc->target_handle,
 			  (int) reloc->offset,
 			  reloc->read_domains,
 			  reloc->write_domain);
@@ -593,39 +1366,57 @@ i915_gem_execbuffer_relocate_entry(struct i915_vma *vma,
 	if (unlikely((reloc->write_domain | reloc->read_domains)
 		     & ~I915_GEM_GPU_DOMAINS)) {
 		DRM_DEBUG("reloc with read/write non-GPU domains: "
-			  "obj %p target %d offset %d "
+			  "target %d offset %d "
 			  "read %08x write %08x",
-			  obj, reloc->target_handle,
+			  reloc->target_handle,
 			  (int) reloc->offset,
 			  reloc->read_domains,
 			  reloc->write_domain);
 		return -EINVAL;
 	}
 
-	target_obj->pending_read_domains |= reloc->read_domains;
-	target_obj->pending_write_domain |= reloc->write_domain;
+	if (reloc->write_domain) {
+		target->exec_entry->flags |= EXEC_OBJECT_WRITE;
+
+		/*
+		 * Sandybridge PPGTT errata: We need a global gtt mapping
+		 * for MI and pipe_control writes because the gpu doesn't
+		 * properly redirect them through the ppgtt for non_secure
+		 * batchbuffers.
+		 */
+		if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
+		    IS_GEN6(eb->i915)) {
+			err = i915_vma_bind(target, target->obj->cache_level,
+					    PIN_GLOBAL);
+			if (WARN_ONCE(err,
+				      "Unexpected failure to bind target VMA!"))
+				return err;
+		}
+	}
 
-	/* If the relocation already has the right value in it, no
+	/*
+	 * If the relocation already has the right value in it, no
 	 * more work needs to be done.
 	 */
-	if (target_offset == reloc->presumed_offset)
+	if (!DBG_FORCE_RELOC &&
+	    gen8_canonical_addr(target->node.start) == reloc->presumed_offset)
 		return 0;
 
 	/* Check that the relocation address is valid... */
 	if (unlikely(reloc->offset >
-		     obj->base.size - (cache->use_64bit_reloc ? 8 : 4))) {
+		     vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
 		DRM_DEBUG("Relocation beyond object bounds: "
-			  "obj %p target %d offset %d size %d.\n",
-			  obj, reloc->target_handle,
-			  (int) reloc->offset,
-			  (int) obj->base.size);
+			  "target %d offset %d size %d.\n",
+			  reloc->target_handle,
+			  (int)reloc->offset,
+			  (int)vma->size);
 		return -EINVAL;
 	}
 	if (unlikely(reloc->offset & 3)) {
 		DRM_DEBUG("Relocation not 4-byte aligned: "
-			  "obj %p target %d offset %d.\n",
-			  obj, reloc->target_handle,
-			  (int) reloc->offset);
+			  "target %d offset %d.\n",
+			  reloc->target_handle,
+			  (int)reloc->offset);
 		return -EINVAL;
 	}
 
@@ -639,39 +1430,39 @@ i915_gem_execbuffer_relocate_entry(struct i915_vma *vma,
 	 */
 	vma->exec_entry->flags &= ~EXEC_OBJECT_ASYNC;
 
-	ret = relocate_entry(obj, reloc, cache, target_offset);
-	if (ret)
-		return ret;
-
 	/* and update the user's relocation entry */
-	reloc->presumed_offset = target_offset;
-	return 0;
+	return relocate_entry(vma, reloc, eb, target);
 }
 
-static int
-i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
-				 struct eb_vmas *eb)
+static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
 {
 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
-	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
-	struct drm_i915_gem_relocation_entry __user *user_relocs;
-	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	struct reloc_cache cache;
-	int remain, ret = 0;
-
-	user_relocs = u64_to_user_ptr(entry->relocs_ptr);
-	reloc_cache_init(&cache, eb->i915);
+	struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
+	struct drm_i915_gem_relocation_entry __user *urelocs;
+	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+	unsigned int remain;
 
+	urelocs = u64_to_user_ptr(entry->relocs_ptr);
 	remain = entry->relocation_count;
-	while (remain) {
-		struct drm_i915_gem_relocation_entry *r = stack_reloc;
-		unsigned long unwritten;
-		unsigned int count;
+	if (unlikely(remain > N_RELOC(ULONG_MAX)))
+		return -EINVAL;
 
-		count = min_t(unsigned int, remain, ARRAY_SIZE(stack_reloc));
-		remain -= count;
+	/*
+	 * We must check that the entire relocation array is safe
+	 * to read. However, if the array is not writable the user loses
+	 * the updated relocation values.
+	 */
+	if (unlikely(!access_ok(VERIFY_READ, urelocs, remain*sizeof(urelocs))))
+		return -EFAULT;
 
-		/* This is the fast path and we cannot handle a pagefault
+	do {
+		struct drm_i915_gem_relocation_entry *r = stack;
+		unsigned int count =
+			min_t(unsigned int, remain, ARRAY_SIZE(stack));
+		unsigned int copied;
+
+		/*
+		 * This is the fast path and we cannot handle a pagefault
 		 * whilst holding the struct mutex lest the user pass in the
 		 * relocations contained within a mmaped bo. For in such a case
 		 * we, the page fault handler would call i915_gem_fault() and
@@ -679,595 +1470,425 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 		 * this is bad and so lockdep complains vehemently.
 		 */
 		pagefault_disable();
-		unwritten = __copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]));
+		copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
 		pagefault_enable();
-		if (unlikely(unwritten)) {
-			ret = -EFAULT;
+		if (unlikely(copied)) {
+			remain = -EFAULT;
 			goto out;
 		}
 
+		remain -= count;
 		do {
-			u64 offset = r->presumed_offset;
+			u64 offset = eb_relocate_entry(eb, vma, r);
 
-			ret = i915_gem_execbuffer_relocate_entry(vma, eb, r, &cache);
-			if (ret)
+			if (likely(offset == 0)) {
+			} else if ((s64)offset < 0) {
+				remain = (int)offset;
 				goto out;
-
-			if (r->presumed_offset != offset) {
-				pagefault_disable();
-				unwritten = __put_user(r->presumed_offset,
-						       &user_relocs->presumed_offset);
-				pagefault_enable();
-				if (unlikely(unwritten)) {
-					/* Note that reporting an error now
-					 * leaves everything in an inconsistent
-					 * state as we have *already* changed
-					 * the relocation value inside the
-					 * object. As we have not changed the
-					 * reloc.presumed_offset or will not
-					 * change the execobject.offset, on the
-					 * call we may not rewrite the value
-					 * inside the object, leaving it
-					 * dangling and causing a GPU hang.
-					 */
-					ret = -EFAULT;
-					goto out;
-				}
+			} else {
+				/*
+				 * Note that reporting an error now
+				 * leaves everything in an inconsistent
+				 * state as we have *already* changed
+				 * the relocation value inside the
+				 * object. As we have not changed the
+				 * reloc.presumed_offset or will not
+				 * change the execobject.offset, on the
+				 * call we may not rewrite the value
+				 * inside the object, leaving it
+				 * dangling and causing a GPU hang. Unless
+				 * userspace dynamically rebuilds the
+				 * relocations on each execbuf rather than
+				 * presume a static tree.
+				 *
+				 * We did previously check if the relocations
+				 * were writable (access_ok), an error now
+				 * would be a strange race with mprotect,
+				 * having already demonstrated that we
+				 * can read from this userspace address.
+				 */
+				offset = gen8_canonical_addr(offset & ~UPDATE);
+				__put_user(offset,
+					   &urelocs[r-stack].presumed_offset);
 			}
-
-			user_relocs++;
-			r++;
-		} while (--count);
-	}
-
+		} while (r++, --count);
+		urelocs += ARRAY_SIZE(stack);
+	} while (remain);
 out:
-	reloc_cache_fini(&cache);
-	return ret;
-#undef N_RELOC
+	reloc_cache_reset(&eb->reloc_cache);
+	return remain;
 }
 
 static int
-i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
-				      struct eb_vmas *eb,
-				      struct drm_i915_gem_relocation_entry *relocs)
+eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
 {
 	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	struct reloc_cache cache;
-	int i, ret = 0;
+	struct drm_i915_gem_relocation_entry *relocs =
+		u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
+	unsigned int i;
+	int err;
 
-	reloc_cache_init(&cache, eb->i915);
 	for (i = 0; i < entry->relocation_count; i++) {
-		ret = i915_gem_execbuffer_relocate_entry(vma, eb, &relocs[i], &cache);
-		if (ret)
-			break;
-	}
-	reloc_cache_fini(&cache);
-
-	return ret;
-}
+		u64 offset = eb_relocate_entry(eb, vma, &relocs[i]);
 
-static int
-i915_gem_execbuffer_relocate(struct eb_vmas *eb)
-{
-	struct i915_vma *vma;
-	int ret = 0;
-
-	list_for_each_entry(vma, &eb->vmas, exec_list) {
-		ret = i915_gem_execbuffer_relocate_vma(vma, eb);
-		if (ret)
-			break;
+		if ((s64)offset < 0) {
+			err = (int)offset;
+			goto err;
+		}
 	}
-
-	return ret;
-}
-
-static bool only_mappable_for_reloc(unsigned int flags)
-{
-	return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
-		__EXEC_OBJECT_NEEDS_MAP;
+	err = 0;
+err:
+	reloc_cache_reset(&eb->reloc_cache);
+	return err;
 }
 
-static int
-i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
-				struct intel_engine_cs *engine,
-				bool *need_reloc)
+static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
 {
-	struct drm_i915_gem_object *obj = vma->obj;
-	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	uint64_t flags;
-	int ret;
+	const char __user *addr, *end;
+	unsigned long size;
+	char __maybe_unused c;
 
-	flags = PIN_USER;
-	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
-		flags |= PIN_GLOBAL;
-
-	if (!drm_mm_node_allocated(&vma->node)) {
-		/* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
-		 * limit address to the first 4GBs for unflagged objects.
-		 */
-		if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
-			flags |= PIN_ZONE_4G;
-		if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
-			flags |= PIN_GLOBAL | PIN_MAPPABLE;
-		if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
-			flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
-		if (entry->flags & EXEC_OBJECT_PINNED)
-			flags |= entry->offset | PIN_OFFSET_FIXED;
-		if ((flags & PIN_MAPPABLE) == 0)
-			flags |= PIN_HIGH;
-	}
-
-	ret = i915_vma_pin(vma,
-			   entry->pad_to_size,
-			   entry->alignment,
-			   flags);
-	if ((ret == -ENOSPC || ret == -E2BIG) &&
-	    only_mappable_for_reloc(entry->flags))
-		ret = i915_vma_pin(vma,
-				   entry->pad_to_size,
-				   entry->alignment,
-				   flags & ~PIN_MAPPABLE);
-	if (ret)
-		return ret;
-
-	entry->flags |= __EXEC_OBJECT_HAS_PIN;
-
-	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
-		ret = i915_vma_get_fence(vma);
-		if (ret)
-			return ret;
+	size = entry->relocation_count;
+	if (size == 0)
+		return 0;
 
-		if (i915_vma_pin_fence(vma))
-			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
-	}
+	if (size > N_RELOC(ULONG_MAX))
+		return -EINVAL;
 
-	if (entry->offset != vma->node.start) {
-		entry->offset = vma->node.start;
-		*need_reloc = true;
-	}
+	addr = u64_to_user_ptr(entry->relocs_ptr);
+	size *= sizeof(struct drm_i915_gem_relocation_entry);
+	if (!access_ok(VERIFY_READ, addr, size))
+		return -EFAULT;
 
-	if (entry->flags & EXEC_OBJECT_WRITE) {
-		obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
-		obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
+	end = addr + size;
+	for (; addr < end; addr += PAGE_SIZE) {
+		int err = __get_user(c, addr);
+		if (err)
+			return err;
 	}
-
-	return 0;
+	return __get_user(c, end - 1);
 }
 
-static bool
-need_reloc_mappable(struct i915_vma *vma)
+static int eb_copy_relocations(const struct i915_execbuffer *eb)
 {
-	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-
-	if (entry->relocation_count == 0)
-		return false;
-
-	if (!i915_vma_is_ggtt(vma))
-		return false;
-
-	/* See also use_cpu_reloc() */
-	if (HAS_LLC(to_i915(vma->obj->base.dev)))
-		return false;
+	const unsigned int count = eb->buffer_count;
+	unsigned int i;
+	int err;
 
-	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
-		return false;
+	for (i = 0; i < count; i++) {
+		const unsigned int nreloc = eb->exec[i].relocation_count;
+		struct drm_i915_gem_relocation_entry __user *urelocs;
+		struct drm_i915_gem_relocation_entry *relocs;
+		unsigned long size;
+		unsigned long copied;
 
-	return true;
-}
+		if (nreloc == 0)
+			continue;
 
-static bool
-eb_vma_misplaced(struct i915_vma *vma)
-{
-	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+		err = check_relocations(&eb->exec[i]);
+		if (err)
+			goto err;
 
-	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
-		!i915_vma_is_ggtt(vma));
+		urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
+		size = nreloc * sizeof(*relocs);
 
-	if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
-		return true;
+		relocs = kvmalloc_array(size, 1, GFP_TEMPORARY);
+		if (!relocs) {
+			kvfree(relocs);
+			err = -ENOMEM;
+			goto err;
+		}
 
-	if (vma->node.size < entry->pad_to_size)
-		return true;
+		/* copy_from_user is limited to < 4GiB */
+		copied = 0;
+		do {
+			unsigned int len =
+				min_t(u64, BIT_ULL(31), size - copied);
+
+			if (__copy_from_user((char *)relocs + copied,
+					     (char *)urelocs + copied,
+					     len)) {
+				kvfree(relocs);
+				err = -EFAULT;
+				goto err;
+			}
 
-	if (entry->flags & EXEC_OBJECT_PINNED &&
-	    vma->node.start != entry->offset)
-		return true;
+			copied += len;
+		} while (copied < size);
 
-	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
-	    vma->node.start < BATCH_OFFSET_BIAS)
-		return true;
+		/*
+		 * As we do not update the known relocation offsets after
+		 * relocating (due to the complexities in lock handling),
+		 * we need to mark them as invalid now so that we force the
+		 * relocation processing next time. Just in case the target
+		 * object is evicted and then rebound into its old
+		 * presumed_offset before the next execbuffer - if that
+		 * happened we would make the mistake of assuming that the
+		 * relocations were valid.
+		 */
+		user_access_begin();
+		for (copied = 0; copied < nreloc; copied++)
+			unsafe_put_user(-1,
+					&urelocs[copied].presumed_offset,
+					end_user);
+end_user:
+		user_access_end();
 
-	/* avoid costly ping-pong once a batch bo ended up non-mappable */
-	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
-	    !i915_vma_is_map_and_fenceable(vma))
-		return !only_mappable_for_reloc(entry->flags);
+		eb->exec[i].relocs_ptr = (uintptr_t)relocs;
+	}
 
-	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
-	    (vma->node.start + vma->node.size - 1) >> 32)
-		return true;
+	return 0;
 
-	return false;
+err:
+	while (i--) {
+		struct drm_i915_gem_relocation_entry *relocs =
+			u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
+		if (eb->exec[i].relocation_count)
+			kvfree(relocs);
+	}
+	return err;
 }
 
-static int
-i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
-			    struct list_head *vmas,
-			    struct i915_gem_context *ctx,
-			    bool *need_relocs)
+static int eb_prefault_relocations(const struct i915_execbuffer *eb)
 {
-	struct drm_i915_gem_object *obj;
-	struct i915_vma *vma;
-	struct i915_address_space *vm;
-	struct list_head ordered_vmas;
-	struct list_head pinned_vmas;
-	bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
-	bool needs_unfenced_map = INTEL_INFO(engine->i915)->unfenced_needs_alignment;
-	int retry;
-
-	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
-
-	INIT_LIST_HEAD(&ordered_vmas);
-	INIT_LIST_HEAD(&pinned_vmas);
-	while (!list_empty(vmas)) {
-		struct drm_i915_gem_exec_object2 *entry;
-		bool need_fence, need_mappable;
-
-		vma = list_first_entry(vmas, struct i915_vma, exec_list);
-		obj = vma->obj;
-		entry = vma->exec_entry;
-
-		if (ctx->flags & CONTEXT_NO_ZEROMAP)
-			entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
-
-		if (!has_fenced_gpu_access)
-			entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
-		need_fence =
-			(entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
-			 needs_unfenced_map) &&
-			i915_gem_object_is_tiled(obj);
-		need_mappable = need_fence || need_reloc_mappable(vma);
-
-		if (entry->flags & EXEC_OBJECT_PINNED)
-			list_move_tail(&vma->exec_list, &pinned_vmas);
-		else if (need_mappable) {
-			entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
-			list_move(&vma->exec_list, &ordered_vmas);
-		} else
-			list_move_tail(&vma->exec_list, &ordered_vmas);
-
-		obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
-		obj->base.pending_write_domain = 0;
-	}
-	list_splice(&ordered_vmas, vmas);
-	list_splice(&pinned_vmas, vmas);
-
-	/* Attempt to pin all of the buffers into the GTT.
-	 * This is done in 3 phases:
-	 *
-	 * 1a. Unbind all objects that do not match the GTT constraints for
-	 *     the execbuffer (fenceable, mappable, alignment etc).
-	 * 1b. Increment pin count for already bound objects.
-	 * 2.  Bind new objects.
-	 * 3.  Decrement pin count.
-	 *
-	 * This avoid unnecessary unbinding of later objects in order to make
-	 * room for the earlier objects *unless* we need to defragment.
-	 */
-	retry = 0;
-	do {
-		int ret = 0;
-
-		/* Unbind any ill-fitting objects or pin. */
-		list_for_each_entry(vma, vmas, exec_list) {
-			if (!drm_mm_node_allocated(&vma->node))
-				continue;
-
-			if (eb_vma_misplaced(vma))
-				ret = i915_vma_unbind(vma);
-			else
-				ret = i915_gem_execbuffer_reserve_vma(vma,
-								      engine,
-								      need_relocs);
-			if (ret)
-				goto err;
-		}
+	const unsigned int count = eb->buffer_count;
+	unsigned int i;
 
-		/* Bind fresh objects */
-		list_for_each_entry(vma, vmas, exec_list) {
-			if (drm_mm_node_allocated(&vma->node))
-				continue;
-
-			ret = i915_gem_execbuffer_reserve_vma(vma, engine,
-							      need_relocs);
-			if (ret)
-				goto err;
-		}
+	if (unlikely(i915.prefault_disable))
+		return 0;
 
-err:
-		if (ret != -ENOSPC || retry++)
-			return ret;
+	for (i = 0; i < count; i++) {
+		int err;
 
-		/* Decrement pin count for bound objects */
-		list_for_each_entry(vma, vmas, exec_list)
-			i915_gem_execbuffer_unreserve_vma(vma);
+		err = check_relocations(&eb->exec[i]);
+		if (err)
+			return err;
+	}
 
-		ret = i915_gem_evict_vm(vm, true);
-		if (ret)
-			return ret;
-	} while (1);
+	return 0;
 }
 
-static int
-i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
-				  struct drm_i915_gem_execbuffer2 *args,
-				  struct drm_file *file,
-				  struct intel_engine_cs *engine,
-				  struct eb_vmas *eb,
-				  struct drm_i915_gem_exec_object2 *exec,
-				  struct i915_gem_context *ctx)
+static noinline int eb_relocate_slow(struct i915_execbuffer *eb)
 {
-	struct drm_i915_gem_relocation_entry *reloc;
-	struct i915_address_space *vm;
+	struct drm_device *dev = &eb->i915->drm;
+	bool have_copy = false;
 	struct i915_vma *vma;
-	bool need_relocs;
-	int *reloc_offset;
-	int i, total, ret;
-	unsigned count = args->buffer_count;
-
-	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
+	int err = 0;
 
-	/* We may process another execbuffer during the unlock... */
-	while (!list_empty(&eb->vmas)) {
-		vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
-		list_del_init(&vma->exec_list);
-		i915_gem_execbuffer_unreserve_vma(vma);
-		i915_vma_put(vma);
+repeat:
+	if (signal_pending(current)) {
+		err = -ERESTARTSYS;
+		goto out;
 	}
 
+	/* We may process another execbuffer during the unlock... */
+	eb_reset_vmas(eb);
 	mutex_unlock(&dev->struct_mutex);
 
-	total = 0;
-	for (i = 0; i < count; i++)
-		total += exec[i].relocation_count;
-
-	reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
-	reloc = drm_malloc_ab(total, sizeof(*reloc));
-	if (reloc == NULL || reloc_offset == NULL) {
-		drm_free_large(reloc);
-		drm_free_large(reloc_offset);
+	/*
+	 * We take 3 passes through the slowpatch.
+	 *
+	 * 1 - we try to just prefault all the user relocation entries and
+	 * then attempt to reuse the atomic pagefault disabled fast path again.
+	 *
+	 * 2 - we copy the user entries to a local buffer here outside of the
+	 * local and allow ourselves to wait upon any rendering before
+	 * relocations
+	 *
+	 * 3 - we already have a local copy of the relocation entries, but
+	 * were interrupted (EAGAIN) whilst waiting for the objects, try again.
+	 */
+	if (!err) {
+		err = eb_prefault_relocations(eb);
+	} else if (!have_copy) {
+		err = eb_copy_relocations(eb);
+		have_copy = err == 0;
+	} else {
+		cond_resched();
+		err = 0;
+	}
+	if (err) {
 		mutex_lock(&dev->struct_mutex);
-		return -ENOMEM;
+		goto out;
 	}
 
-	total = 0;
-	for (i = 0; i < count; i++) {
-		struct drm_i915_gem_relocation_entry __user *user_relocs;
-		u64 invalid_offset = (u64)-1;
-		int j;
-
-		user_relocs = u64_to_user_ptr(exec[i].relocs_ptr);
-
-		if (copy_from_user(reloc+total, user_relocs,
-				   exec[i].relocation_count * sizeof(*reloc))) {
-			ret = -EFAULT;
-			mutex_lock(&dev->struct_mutex);
-			goto err;
-		}
+	/* A frequent cause for EAGAIN are currently unavailable client pages */
+	flush_workqueue(eb->i915->mm.userptr_wq);
 
-		/* As we do not update the known relocation offsets after
-		 * relocating (due to the complexities in lock handling),
-		 * we need to mark them as invalid now so that we force the
-		 * relocation processing next time. Just in case the target
-		 * object is evicted and then rebound into its old
-		 * presumed_offset before the next execbuffer - if that
-		 * happened we would make the mistake of assuming that the
-		 * relocations were valid.
-		 */
-		for (j = 0; j < exec[i].relocation_count; j++) {
-			if (__copy_to_user(&user_relocs[j].presumed_offset,
-					   &invalid_offset,
-					   sizeof(invalid_offset))) {
-				ret = -EFAULT;
-				mutex_lock(&dev->struct_mutex);
-				goto err;
-			}
-		}
-
-		reloc_offset[i] = total;
-		total += exec[i].relocation_count;
-	}
-
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret) {
+	err = i915_mutex_lock_interruptible(dev);
+	if (err) {
 		mutex_lock(&dev->struct_mutex);
-		goto err;
+		goto out;
 	}
 
 	/* reacquire the objects */
-	eb_reset(eb);
-	ret = eb_lookup_vmas(eb, exec, args, vm, file);
-	if (ret)
-		goto err;
-
-	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
-	ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
-					  &need_relocs);
-	if (ret)
+	err = eb_lookup_vmas(eb);
+	if (err)
 		goto err;
 
-	list_for_each_entry(vma, &eb->vmas, exec_list) {
-		int offset = vma->exec_entry - exec;
-		ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
-							    reloc + reloc_offset[offset]);
-		if (ret)
-			goto err;
+	list_for_each_entry(vma, &eb->relocs, reloc_link) {
+		if (!have_copy) {
+			pagefault_disable();
+			err = eb_relocate_vma(eb, vma);
+			pagefault_enable();
+			if (err)
+				goto repeat;
+		} else {
+			err = eb_relocate_vma_slow(eb, vma);
+			if (err)
+				goto err;
+		}
 	}
 
-	/* Leave the user relocations as are, this is the painfully slow path,
+	/*
+	 * Leave the user relocations as are, this is the painfully slow path,
 	 * and we want to avoid the complication of dropping the lock whilst
 	 * having buffers reserved in the aperture and so causing spurious
 	 * ENOSPC for random operations.
 	 */
 
 err:
-	drm_free_large(reloc);
-	drm_free_large(reloc_offset);
-	return ret;
-}
+	if (err == -EAGAIN)
+		goto repeat;
 
-static int
-i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
-				struct list_head *vmas)
-{
-	struct i915_vma *vma;
-	int ret;
+out:
+	if (have_copy) {
+		const unsigned int count = eb->buffer_count;
+		unsigned int i;
 
-	list_for_each_entry(vma, vmas, exec_list) {
-		struct drm_i915_gem_object *obj = vma->obj;
+		for (i = 0; i < count; i++) {
+			const struct drm_i915_gem_exec_object2 *entry =
+				&eb->exec[i];
+			struct drm_i915_gem_relocation_entry *relocs;
 
-		if (vma->exec_entry->flags & EXEC_OBJECT_ASYNC)
-			continue;
+			if (!entry->relocation_count)
+				continue;
 
-		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) {
-			i915_gem_clflush_object(obj, 0);
-			obj->base.write_domain = 0;
+			relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
+			kvfree(relocs);
 		}
-
-		ret = i915_gem_request_await_object
-			(req, obj, obj->base.pending_write_domain);
-		if (ret)
-			return ret;
 	}
 
-	/* Unconditionally flush any chipset caches (for streaming writes). */
-	i915_gem_chipset_flush(req->engine->i915);
-
-	/* Unconditionally invalidate GPU caches and TLBs. */
-	return req->engine->emit_flush(req, EMIT_INVALIDATE);
+	return err ?: have_copy;
 }
 
-static bool
-i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
+static int eb_relocate(struct i915_execbuffer *eb)
 {
-	if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
-		return false;
+	if (eb_lookup_vmas(eb))
+		goto slow;
 
-	/* Kernel clipping was a DRI1 misfeature */
-	if (exec->num_cliprects || exec->cliprects_ptr)
-		return false;
+	/* The objects are in their final locations, apply the relocations. */
+	if (eb->args->flags & __EXEC_HAS_RELOC) {
+		struct i915_vma *vma;
 
-	if (exec->DR4 == 0xffffffff) {
-		DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
-		exec->DR4 = 0;
+		list_for_each_entry(vma, &eb->relocs, reloc_link) {
+			if (eb_relocate_vma(eb, vma))
+				goto slow;
+		}
 	}
-	if (exec->DR1 || exec->DR4)
-		return false;
 
-	if ((exec->batch_start_offset | exec->batch_len) & 0x7)
-		return false;
+	return 0;
 
-	return true;
+slow:
+	return eb_relocate_slow(eb);
 }
 
-static int
-validate_exec_list(struct drm_device *dev,
-		   struct drm_i915_gem_exec_object2 *exec,
-		   int count)
+static void eb_export_fence(struct i915_vma *vma,
+			    struct drm_i915_gem_request *req,
+			    unsigned int flags)
 {
-	unsigned relocs_total = 0;
-	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
-	unsigned invalid_flags;
-	int i;
+	struct reservation_object *resv = vma->resv;
 
-	/* INTERNAL flags must not overlap with external ones */
-	BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
+	/*
+	 * Ignore errors from failing to allocate the new fence, we can't
+	 * handle an error right now. Worst case should be missed
+	 * synchronisation leading to rendering corruption.
+	 */
+	reservation_object_lock(resv, NULL);
+	if (flags & EXEC_OBJECT_WRITE)
+		reservation_object_add_excl_fence(resv, &req->fence);
+	else if (reservation_object_reserve_shared(resv) == 0)
+		reservation_object_add_shared_fence(resv, &req->fence);
+	reservation_object_unlock(resv);
+}
 
-	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
-	if (USES_FULL_PPGTT(dev))
-		invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
+static int eb_move_to_gpu(struct i915_execbuffer *eb)
+{
+	const unsigned int count = eb->buffer_count;
+	unsigned int i;
+	int err;
 
 	for (i = 0; i < count; i++) {
-		char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
-		int length; /* limited by fault_in_pages_readable() */
+		const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+		struct i915_vma *vma = exec_to_vma(entry);
+		struct drm_i915_gem_object *obj = vma->obj;
 
-		if (exec[i].flags & invalid_flags)
-			return -EINVAL;
+		if (entry->flags & EXEC_OBJECT_CAPTURE) {
+			struct i915_gem_capture_list *capture;
 
-		/* Offset can be used as input (EXEC_OBJECT_PINNED), reject
-		 * any non-page-aligned or non-canonical addresses.
-		 */
-		if (exec[i].flags & EXEC_OBJECT_PINNED) {
-			if (exec[i].offset !=
-			    gen8_canonical_addr(exec[i].offset & PAGE_MASK))
-				return -EINVAL;
+			capture = kmalloc(sizeof(*capture), GFP_KERNEL);
+			if (unlikely(!capture))
+				return -ENOMEM;
+
+			capture->next = eb->request->capture_list;
+			capture->vma = vma;
+			eb->request->capture_list = capture;
 		}
 
-		/* From drm_mm perspective address space is continuous,
-		 * so from this point we're always using non-canonical
-		 * form internally.
-		 */
-		exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
+		if (entry->flags & EXEC_OBJECT_ASYNC)
+			goto skip_flushes;
 
-		if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
-			return -EINVAL;
+		if (unlikely(obj->cache_dirty && !obj->cache_coherent))
+			i915_gem_clflush_object(obj, 0);
 
-		/* pad_to_size was once a reserved field, so sanitize it */
-		if (exec[i].flags & EXEC_OBJECT_PAD_TO_SIZE) {
-			if (offset_in_page(exec[i].pad_to_size))
-				return -EINVAL;
-		} else {
-			exec[i].pad_to_size = 0;
-		}
+		err = i915_gem_request_await_object
+			(eb->request, obj, entry->flags & EXEC_OBJECT_WRITE);
+		if (err)
+			return err;
 
-		/* First check for malicious input causing overflow in
-		 * the worst case where we need to allocate the entire
-		 * relocation tree as a single array.
-		 */
-		if (exec[i].relocation_count > relocs_max - relocs_total)
-			return -EINVAL;
-		relocs_total += exec[i].relocation_count;
+skip_flushes:
+		i915_vma_move_to_active(vma, eb->request, entry->flags);
+		__eb_unreserve_vma(vma, entry);
+		vma->exec_entry = NULL;
+	}
 
-		length = exec[i].relocation_count *
-			sizeof(struct drm_i915_gem_relocation_entry);
-		/*
-		 * We must check that the entire relocation array is safe
-		 * to read, but since we may need to update the presumed
-		 * offsets during execution, check for full write access.
-		 */
-		if (!access_ok(VERIFY_WRITE, ptr, length))
-			return -EFAULT;
+	for (i = 0; i < count; i++) {
+		const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+		struct i915_vma *vma = exec_to_vma(entry);
 
-		if (likely(!i915.prefault_disable)) {
-			if (fault_in_pages_readable(ptr, length))
-				return -EFAULT;
-		}
+		eb_export_fence(vma, eb->request, entry->flags);
+		if (unlikely(entry->flags & __EXEC_OBJECT_HAS_REF))
+			i915_vma_put(vma);
 	}
+	eb->exec = NULL;
 
-	return 0;
+	/* Unconditionally flush any chipset caches (for streaming writes). */
+	i915_gem_chipset_flush(eb->i915);
+
+	/* Unconditionally invalidate GPU caches and TLBs. */
+	return eb->engine->emit_flush(eb->request, EMIT_INVALIDATE);
 }
 
-static struct i915_gem_context *
-i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
-			  struct intel_engine_cs *engine, const u32 ctx_id)
+static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 {
-	struct i915_gem_context *ctx;
+	if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
+		return false;
 
-	ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
-	if (IS_ERR(ctx))
-		return ctx;
+	/* Kernel clipping was a DRI1 misfeature */
+	if (exec->num_cliprects || exec->cliprects_ptr)
+		return false;
 
-	if (i915_gem_context_is_banned(ctx)) {
-		DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
-		return ERR_PTR(-EIO);
+	if (exec->DR4 == 0xffffffff) {
+		DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
+		exec->DR4 = 0;
 	}
+	if (exec->DR1 || exec->DR4)
+		return false;
 
-	return ctx;
-}
+	if ((exec->batch_start_offset | exec->batch_len) & 0x7)
+		return false;
 
-static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
-{
-	return !(obj->cache_level == I915_CACHE_NONE ||
-		 obj->cache_level == I915_CACHE_WT);
+	return true;
 }
 
 void i915_vma_move_to_active(struct i915_vma *vma,
@@ -1280,7 +1901,8 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 	lockdep_assert_held(&req->i915->drm.struct_mutex);
 	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
-	/* Add a reference if we're newly entering the active list.
+	/*
+	 * Add a reference if we're newly entering the active list.
 	 * The order in which we add operations to the retirement queue is
 	 * vital here: mark_active adds to the start of the callback list,
 	 * such that subsequent callbacks are called first. Therefore we
@@ -1293,61 +1915,22 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 	i915_gem_active_set(&vma->last_read[idx], req);
 	list_move_tail(&vma->vm_link, &vma->vm->active_list);
 
+	obj->base.write_domain = 0;
 	if (flags & EXEC_OBJECT_WRITE) {
+		obj->base.write_domain = I915_GEM_DOMAIN_RENDER;
+
 		if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
 			i915_gem_active_set(&obj->frontbuffer_write, req);
 
-		/* update for the implicit flush after a batch */
-		obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
-		if (!obj->cache_dirty && gpu_write_needs_clflush(obj))
-			obj->cache_dirty = true;
+		obj->base.read_domains = 0;
 	}
+	obj->base.read_domains |= I915_GEM_GPU_DOMAINS;
 
 	if (flags & EXEC_OBJECT_NEEDS_FENCE)
 		i915_gem_active_set(&vma->last_fence, req);
 }
 
-static void eb_export_fence(struct drm_i915_gem_object *obj,
-			    struct drm_i915_gem_request *req,
-			    unsigned int flags)
-{
-	struct reservation_object *resv = obj->resv;
-
-	/* Ignore errors from failing to allocate the new fence, we can't
-	 * handle an error right now. Worst case should be missed
-	 * synchronisation leading to rendering corruption.
-	 */
-	reservation_object_lock(resv, NULL);
-	if (flags & EXEC_OBJECT_WRITE)
-		reservation_object_add_excl_fence(resv, &req->fence);
-	else if (reservation_object_reserve_shared(resv) == 0)
-		reservation_object_add_shared_fence(resv, &req->fence);
-	reservation_object_unlock(resv);
-}
-
-static void
-i915_gem_execbuffer_move_to_active(struct list_head *vmas,
-				   struct drm_i915_gem_request *req)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, vmas, exec_list) {
-		struct drm_i915_gem_object *obj = vma->obj;
-
-		obj->base.write_domain = obj->base.pending_write_domain;
-		if (obj->base.write_domain)
-			vma->exec_entry->flags |= EXEC_OBJECT_WRITE;
-		else
-			obj->base.pending_read_domains |= obj->base.read_domains;
-		obj->base.read_domains = obj->base.pending_read_domains;
-
-		i915_vma_move_to_active(vma, req, vma->exec_entry->flags);
-		eb_export_fence(obj, req, vma->exec_entry->flags);
-	}
-}
-
-static int
-i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
+static int i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
 {
 	u32 *cs;
 	int i;
@@ -1357,50 +1940,43 @@ i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
 		return -EINVAL;
 	}
 
-	cs = intel_ring_begin(req, 4 * 3);
+	cs = intel_ring_begin(req, 4 * 2 + 2);
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
 
+	*cs++ = MI_LOAD_REGISTER_IMM(4);
 	for (i = 0; i < 4; i++) {
-		*cs++ = MI_LOAD_REGISTER_IMM(1);
 		*cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
 		*cs++ = 0;
 	}
-
+	*cs++ = MI_NOOP;
 	intel_ring_advance(req, cs);
 
 	return 0;
 }
 
-static struct i915_vma *
-i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
-			  struct drm_i915_gem_exec_object2 *shadow_exec_entry,
-			  struct drm_i915_gem_object *batch_obj,
-			  struct eb_vmas *eb,
-			  u32 batch_start_offset,
-			  u32 batch_len,
-			  bool is_master)
+static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
 {
 	struct drm_i915_gem_object *shadow_batch_obj;
 	struct i915_vma *vma;
-	int ret;
+	int err;
 
-	shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
-						   PAGE_ALIGN(batch_len));
+	shadow_batch_obj = i915_gem_batch_pool_get(&eb->engine->batch_pool,
+						   PAGE_ALIGN(eb->batch_len));
 	if (IS_ERR(shadow_batch_obj))
 		return ERR_CAST(shadow_batch_obj);
 
-	ret = intel_engine_cmd_parser(engine,
-				      batch_obj,
+	err = intel_engine_cmd_parser(eb->engine,
+				      eb->batch->obj,
 				      shadow_batch_obj,
-				      batch_start_offset,
-				      batch_len,
+				      eb->batch_start_offset,
+				      eb->batch_len,
 				      is_master);
-	if (ret) {
-		if (ret == -EACCES) /* unhandled chained batch */
+	if (err) {
+		if (err == -EACCES) /* unhandled chained batch */
 			vma = NULL;
 		else
-			vma = ERR_PTR(ret);
+			vma = ERR_PTR(err);
 		goto out;
 	}
 
@@ -1408,12 +1984,11 @@ i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
 	if (IS_ERR(vma))
 		goto out;
 
-	memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
-
-	vma->exec_entry = shadow_exec_entry;
-	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
-	i915_gem_object_get(shadow_batch_obj);
-	list_add_tail(&vma->exec_list, &eb->vmas);
+	vma->exec_entry =
+		memset(&eb->exec[eb->buffer_count++],
+		       0, sizeof(*vma->exec_entry));
+	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
+	__exec_to_vma(vma->exec_entry) = (uintptr_t)i915_vma_get(vma);
 
 out:
 	i915_gem_object_unpin_pages(shadow_batch_obj);
@@ -1421,54 +1996,37 @@ out:
 }
 
 static void
-add_to_client(struct drm_i915_gem_request *req,
-	      struct drm_file *file)
+add_to_client(struct drm_i915_gem_request *req, struct drm_file *file)
 {
 	req->file_priv = file->driver_priv;
 	list_add_tail(&req->client_link, &req->file_priv->mm.request_list);
 }
 
-static int
-execbuf_submit(struct i915_execbuffer_params *params,
-	       struct drm_i915_gem_execbuffer2 *args,
-	       struct list_head *vmas)
+static int eb_submit(struct i915_execbuffer *eb)
 {
-	u64 exec_start, exec_len;
-	int ret;
-
-	ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
-	if (ret)
-		return ret;
+	int err;
 
-	ret = i915_switch_context(params->request);
-	if (ret)
-		return ret;
+	err = eb_move_to_gpu(eb);
+	if (err)
+		return err;
 
-	if (args->flags & I915_EXEC_CONSTANTS_MASK) {
-		DRM_DEBUG("I915_EXEC_CONSTANTS_* unsupported\n");
-		return -EINVAL;
-	}
+	err = i915_switch_context(eb->request);
+	if (err)
+		return err;
 
-	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
-		ret = i915_reset_gen7_sol_offsets(params->request);
-		if (ret)
-			return ret;
+	if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
+		err = i915_reset_gen7_sol_offsets(eb->request);
+		if (err)
+			return err;
 	}
 
-	exec_len   = args->batch_len;
-	exec_start = params->batch->node.start +
-		     params->args_batch_start_offset;
-
-	if (exec_len == 0)
-		exec_len = params->batch->size - params->args_batch_start_offset;
-
-	ret = params->engine->emit_bb_start(params->request,
-					    exec_start, exec_len,
-					    params->dispatch_flags);
-	if (ret)
-		return ret;
-
-	i915_gem_execbuffer_move_to_active(vmas, params->request);
+	err = eb->engine->emit_bb_start(eb->request,
+					eb->batch->node.start +
+					eb->batch_start_offset,
+					eb->batch_len,
+					eb->batch_flags);
+	if (err)
+		return err;
 
 	return 0;
 }
@@ -1550,66 +2108,62 @@ eb_select_engine(struct drm_i915_private *dev_priv,
 }
 
 static int
-i915_gem_do_execbuffer(struct drm_device *dev, void *data,
+i915_gem_do_execbuffer(struct drm_device *dev,
 		       struct drm_file *file,
 		       struct drm_i915_gem_execbuffer2 *args,
 		       struct drm_i915_gem_exec_object2 *exec)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct eb_vmas *eb;
-	struct drm_i915_gem_exec_object2 shadow_exec_entry;
-	struct intel_engine_cs *engine;
-	struct i915_gem_context *ctx;
-	struct i915_address_space *vm;
-	struct i915_execbuffer_params params_master; /* XXX: will be removed later */
-	struct i915_execbuffer_params *params = &params_master;
-	const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
-	u32 dispatch_flags;
+	struct i915_execbuffer eb;
 	struct dma_fence *in_fence = NULL;
 	struct sync_file *out_fence = NULL;
 	int out_fence_fd = -1;
-	int ret;
-	bool need_relocs;
-
-	if (!i915_gem_check_execbuffer(args))
-		return -EINVAL;
-
-	ret = validate_exec_list(dev, exec, args->buffer_count);
-	if (ret)
-		return ret;
-
-	dispatch_flags = 0;
+	int err;
+
+	BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
+		     ~__EXEC_OBJECT_UNKNOWN_FLAGS);
+
+	eb.i915 = to_i915(dev);
+	eb.file = file;
+	eb.args = args;
+	if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
+		args->flags |= __EXEC_HAS_RELOC;
+	eb.exec = exec;
+	eb.ctx = NULL;
+	eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
+	if (USES_FULL_PPGTT(eb.i915))
+		eb.invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
+	reloc_cache_init(&eb.reloc_cache, eb.i915);
+
+	eb.buffer_count = args->buffer_count;
+	eb.batch_start_offset = args->batch_start_offset;
+	eb.batch_len = args->batch_len;
+
+	eb.batch_flags = 0;
 	if (args->flags & I915_EXEC_SECURE) {
 		if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
 		    return -EPERM;
 
-		dispatch_flags |= I915_DISPATCH_SECURE;
+		eb.batch_flags |= I915_DISPATCH_SECURE;
 	}
 	if (args->flags & I915_EXEC_IS_PINNED)
-		dispatch_flags |= I915_DISPATCH_PINNED;
+		eb.batch_flags |= I915_DISPATCH_PINNED;
 
-	engine = eb_select_engine(dev_priv, file, args);
-	if (!engine)
+	eb.engine = eb_select_engine(eb.i915, file, args);
+	if (!eb.engine)
 		return -EINVAL;
 
-	if (args->buffer_count < 1) {
-		DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
-		return -EINVAL;
-	}
-
 	if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
-		if (!HAS_RESOURCE_STREAMER(dev_priv)) {
+		if (!HAS_RESOURCE_STREAMER(eb.i915)) {
 			DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
 			return -EINVAL;
 		}
-		if (engine->id != RCS) {
+		if (eb.engine->id != RCS) {
 			DRM_DEBUG("RS is not available on %s\n",
-				 engine->name);
+				 eb.engine->name);
 			return -EINVAL;
 		}
 
-		dispatch_flags |= I915_DISPATCH_RS;
+		eb.batch_flags |= I915_DISPATCH_RS;
 	}
 
 	if (args->flags & I915_EXEC_FENCE_IN) {
@@ -1621,102 +2175,62 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 	if (args->flags & I915_EXEC_FENCE_OUT) {
 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
 		if (out_fence_fd < 0) {
-			ret = out_fence_fd;
+			err = out_fence_fd;
 			goto err_in_fence;
 		}
 	}
 
-	/* Take a local wakeref for preparing to dispatch the execbuf as
+	if (eb_create(&eb))
+		return -ENOMEM;
+
+	/*
+	 * Take a local wakeref for preparing to dispatch the execbuf as
 	 * we expect to access the hardware fairly frequently in the
 	 * process. Upon first dispatch, we acquire another prolonged
 	 * wakeref that we hold until the GPU has been idle for at least
 	 * 100ms.
 	 */
-	intel_runtime_pm_get(dev_priv);
-
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		goto pre_mutex_err;
+	intel_runtime_pm_get(eb.i915);
+	err = i915_mutex_lock_interruptible(dev);
+	if (err)
+		goto err_rpm;
 
-	ctx = i915_gem_validate_context(dev, file, engine, ctx_id);
-	if (IS_ERR(ctx)) {
-		mutex_unlock(&dev->struct_mutex);
-		ret = PTR_ERR(ctx);
-		goto pre_mutex_err;
-	}
-
-	i915_gem_context_get(ctx);
-
-	if (ctx->ppgtt)
-		vm = &ctx->ppgtt->base;
-	else
-		vm = &ggtt->base;
-
-	memset(&params_master, 0x00, sizeof(params_master));
+	err = eb_select_context(&eb);
+	if (unlikely(err))
+		goto err_unlock;
 
-	eb = eb_create(dev_priv, args);
-	if (eb == NULL) {
-		i915_gem_context_put(ctx);
-		mutex_unlock(&dev->struct_mutex);
-		ret = -ENOMEM;
-		goto pre_mutex_err;
-	}
-
-	/* Look up object handles */
-	ret = eb_lookup_vmas(eb, exec, args, vm, file);
-	if (ret)
-		goto err;
-
-	/* take note of the batch buffer before we might reorder the lists */
-	params->batch = eb_get_batch(eb);
-
-	/* Move the objects en-masse into the GTT, evicting if necessary. */
-	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
-	ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
-					  &need_relocs);
-	if (ret)
-		goto err;
-
-	/* The objects are in their final locations, apply the relocations. */
-	if (need_relocs)
-		ret = i915_gem_execbuffer_relocate(eb);
-	if (ret) {
-		if (ret == -EFAULT) {
-			ret = i915_gem_execbuffer_relocate_slow(dev, args, file,
-								engine,
-								eb, exec, ctx);
-			BUG_ON(!mutex_is_locked(&dev->struct_mutex));
-		}
-		if (ret)
-			goto err;
-	}
+	err = eb_relocate(&eb);
+	if (err)
+		/*
+		 * If the user expects the execobject.offset and
+		 * reloc.presumed_offset to be an exact match,
+		 * as for using NO_RELOC, then we cannot update
+		 * the execobject.offset until we have completed
+		 * relocation.
+		 */
+		args->flags &= ~__EXEC_HAS_RELOC;
+	if (err < 0)
+		goto err_vma;
 
-	/* Set the pending read domains for the batch buffer to COMMAND */
-	if (params->batch->obj->base.pending_write_domain) {
+	if (unlikely(eb.batch->exec_entry->flags & EXEC_OBJECT_WRITE)) {
 		DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
-		ret = -EINVAL;
-		goto err;
+		err = -EINVAL;
+		goto err_vma;
 	}
-	if (args->batch_start_offset > params->batch->size ||
-	    args->batch_len > params->batch->size - args->batch_start_offset) {
+	if (eb.batch_start_offset > eb.batch->size ||
+	    eb.batch_len > eb.batch->size - eb.batch_start_offset) {
 		DRM_DEBUG("Attempting to use out-of-bounds batch\n");
-		ret = -EINVAL;
-		goto err;
+		err = -EINVAL;
+		goto err_vma;
 	}
 
-	params->args_batch_start_offset = args->batch_start_offset;
-	if (engine->needs_cmd_parser && args->batch_len) {
+	if (eb.engine->needs_cmd_parser && eb.batch_len) {
 		struct i915_vma *vma;
 
-		vma = i915_gem_execbuffer_parse(engine, &shadow_exec_entry,
-						params->batch->obj,
-						eb,
-						args->batch_start_offset,
-						args->batch_len,
-						drm_is_current_master(file));
+		vma = eb_parse(&eb, drm_is_current_master(file));
 		if (IS_ERR(vma)) {
-			ret = PTR_ERR(vma);
-			goto err;
+			err = PTR_ERR(vma);
+			goto err_vma;
 		}
 
 		if (vma) {
@@ -1729,19 +2243,20 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 			 * specifically don't want that set on batches the
 			 * command parser has accepted.
 			 */
-			dispatch_flags |= I915_DISPATCH_SECURE;
-			params->args_batch_start_offset = 0;
-			params->batch = vma;
+			eb.batch_flags |= I915_DISPATCH_SECURE;
+			eb.batch_start_offset = 0;
+			eb.batch = vma;
 		}
 	}
 
-	params->batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
+	if (eb.batch_len == 0)
+		eb.batch_len = eb.batch->size - eb.batch_start_offset;
 
-	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
+	/*
+	 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
 	 * batch" bit. Hence we need to pin secure batches into the global gtt.
 	 * hsw should have this fixed, but bdw mucks it up again. */
-	if (dispatch_flags & I915_DISPATCH_SECURE) {
-		struct drm_i915_gem_object *obj = params->batch->obj;
+	if (eb.batch_flags & I915_DISPATCH_SECURE) {
 		struct i915_vma *vma;
 
 		/*
@@ -1754,66 +2269,56 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		 *   fitting due to fragmentation.
 		 * So this is actually safe.
 		 */
-		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+		vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0);
 		if (IS_ERR(vma)) {
-			ret = PTR_ERR(vma);
-			goto err;
+			err = PTR_ERR(vma);
+			goto err_vma;
 		}
 
-		params->batch = vma;
+		eb.batch = vma;
 	}
 
+	/* All GPU relocation batches must be submitted prior to the user rq */
+	GEM_BUG_ON(eb.reloc_cache.rq);
+
 	/* Allocate a request for this batch buffer nice and early. */
-	params->request = i915_gem_request_alloc(engine, ctx);
-	if (IS_ERR(params->request)) {
-		ret = PTR_ERR(params->request);
+	eb.request = i915_gem_request_alloc(eb.engine, eb.ctx);
+	if (IS_ERR(eb.request)) {
+		err = PTR_ERR(eb.request);
 		goto err_batch_unpin;
 	}
 
 	if (in_fence) {
-		ret = i915_gem_request_await_dma_fence(params->request,
-						       in_fence);
-		if (ret < 0)
+		err = i915_gem_request_await_dma_fence(eb.request, in_fence);
+		if (err < 0)
 			goto err_request;
 	}
 
 	if (out_fence_fd != -1) {
-		out_fence = sync_file_create(&params->request->fence);
+		out_fence = sync_file_create(&eb.request->fence);
 		if (!out_fence) {
-			ret = -ENOMEM;
+			err = -ENOMEM;
 			goto err_request;
 		}
 	}
 
-	/* Whilst this request exists, batch_obj will be on the
+	/*
+	 * Whilst this request exists, batch_obj will be on the
 	 * active_list, and so will hold the active reference. Only when this
 	 * request is retired will the the batch_obj be moved onto the
 	 * inactive_list and lose its active reference. Hence we do not need
 	 * to explicitly hold another reference here.
 	 */
-	params->request->batch = params->batch;
-
-	/*
-	 * Save assorted stuff away to pass through to *_submission().
-	 * NB: This data should be 'persistent' and not local as it will
-	 * kept around beyond the duration of the IOCTL once the GPU
-	 * scheduler arrives.
-	 */
-	params->dev                     = dev;
-	params->file                    = file;
-	params->engine                    = engine;
-	params->dispatch_flags          = dispatch_flags;
-	params->ctx                     = ctx;
-
-	trace_i915_gem_request_queue(params->request, dispatch_flags);
+	eb.request->batch = eb.batch;
 
-	ret = execbuf_submit(params, args, &eb->vmas);
+	trace_i915_gem_request_queue(eb.request, eb.batch_flags);
+	err = eb_submit(&eb);
 err_request:
-	__i915_add_request(params->request, ret == 0);
-	add_to_client(params->request, file);
+	__i915_add_request(eb.request, err == 0);
+	add_to_client(eb.request, file);
 
 	if (out_fence) {
-		if (ret == 0) {
+		if (err == 0) {
 			fd_install(out_fence_fd, out_fence->file);
 			args->rsvd2 &= GENMASK_ULL(0, 31); /* keep in-fence */
 			args->rsvd2 |= (u64)out_fence_fd << 32;
@@ -1824,30 +2329,22 @@ err_request:
 	}
 
 err_batch_unpin:
-	/*
-	 * FIXME: We crucially rely upon the active tracking for the (ppgtt)
-	 * batch vma for correctness. For less ugly and less fragility this
-	 * needs to be adjusted to also track the ggtt batch vma properly as
-	 * active.
-	 */
-	if (dispatch_flags & I915_DISPATCH_SECURE)
-		i915_vma_unpin(params->batch);
-err:
-	/* the request owns the ref now */
-	i915_gem_context_put(ctx);
-	eb_destroy(eb);
-
+	if (eb.batch_flags & I915_DISPATCH_SECURE)
+		i915_vma_unpin(eb.batch);
+err_vma:
+	if (eb.exec)
+		eb_release_vmas(&eb);
+	i915_gem_context_put(eb.ctx);
+err_unlock:
 	mutex_unlock(&dev->struct_mutex);
-
-pre_mutex_err:
-	/* intel_gpu_busy should also get a ref, so it will free when the device
-	 * is really idle. */
-	intel_runtime_pm_put(dev_priv);
+err_rpm:
+	intel_runtime_pm_put(eb.i915);
+	eb_destroy(&eb);
 	if (out_fence_fd != -1)
 		put_unused_fd(out_fence_fd);
 err_in_fence:
 	dma_fence_put(in_fence);
-	return ret;
+	return err;
 }
 
 /*
@@ -1858,35 +2355,53 @@ int
 i915_gem_execbuffer(struct drm_device *dev, void *data,
 		    struct drm_file *file)
 {
+	const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
 	struct drm_i915_gem_execbuffer *args = data;
 	struct drm_i915_gem_execbuffer2 exec2;
 	struct drm_i915_gem_exec_object *exec_list = NULL;
 	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-	int ret, i;
+	unsigned int i;
+	int err;
 
-	if (args->buffer_count < 1) {
-		DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
+	if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
+		DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
 		return -EINVAL;
 	}
 
+	exec2.buffers_ptr = args->buffers_ptr;
+	exec2.buffer_count = args->buffer_count;
+	exec2.batch_start_offset = args->batch_start_offset;
+	exec2.batch_len = args->batch_len;
+	exec2.DR1 = args->DR1;
+	exec2.DR4 = args->DR4;
+	exec2.num_cliprects = args->num_cliprects;
+	exec2.cliprects_ptr = args->cliprects_ptr;
+	exec2.flags = I915_EXEC_RENDER;
+	i915_execbuffer2_set_context_id(exec2, 0);
+
+	if (!i915_gem_check_execbuffer(&exec2))
+		return -EINVAL;
+
 	/* Copy in the exec list from userland */
-	exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
-	exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
+	exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
+				   __GFP_NOWARN | GFP_TEMPORARY);
+	exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+				    __GFP_NOWARN | GFP_TEMPORARY);
 	if (exec_list == NULL || exec2_list == NULL) {
 		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
 			  args->buffer_count);
-		drm_free_large(exec_list);
-		drm_free_large(exec2_list);
+		kvfree(exec_list);
+		kvfree(exec2_list);
 		return -ENOMEM;
 	}
-	ret = copy_from_user(exec_list,
+	err = copy_from_user(exec_list,
 			     u64_to_user_ptr(args->buffers_ptr),
 			     sizeof(*exec_list) * args->buffer_count);
-	if (ret != 0) {
+	if (err) {
 		DRM_DEBUG("copy %d exec entries failed %d\n",
-			  args->buffer_count, ret);
-		drm_free_large(exec_list);
-		drm_free_large(exec2_list);
+			  args->buffer_count, err);
+		kvfree(exec_list);
+		kvfree(exec2_list);
 		return -EFAULT;
 	}
 
@@ -1902,99 +2417,94 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 			exec2_list[i].flags = 0;
 	}
 
-	exec2.buffers_ptr = args->buffers_ptr;
-	exec2.buffer_count = args->buffer_count;
-	exec2.batch_start_offset = args->batch_start_offset;
-	exec2.batch_len = args->batch_len;
-	exec2.DR1 = args->DR1;
-	exec2.DR4 = args->DR4;
-	exec2.num_cliprects = args->num_cliprects;
-	exec2.cliprects_ptr = args->cliprects_ptr;
-	exec2.flags = I915_EXEC_RENDER;
-	i915_execbuffer2_set_context_id(exec2, 0);
-
-	ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
-	if (!ret) {
+	err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
+	if (exec2.flags & __EXEC_HAS_RELOC) {
 		struct drm_i915_gem_exec_object __user *user_exec_list =
 			u64_to_user_ptr(args->buffers_ptr);
 
 		/* Copy the new buffer offsets back to the user's exec list. */
 		for (i = 0; i < args->buffer_count; i++) {
+			if (!(exec2_list[i].offset & UPDATE))
+				continue;
+
 			exec2_list[i].offset =
-				gen8_canonical_addr(exec2_list[i].offset);
-			ret = __copy_to_user(&user_exec_list[i].offset,
-					     &exec2_list[i].offset,
-					     sizeof(user_exec_list[i].offset));
-			if (ret) {
-				ret = -EFAULT;
-				DRM_DEBUG("failed to copy %d exec entries "
-					  "back to user (%d)\n",
-					  args->buffer_count, ret);
+				gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
+			exec2_list[i].offset &= PIN_OFFSET_MASK;
+			if (__copy_to_user(&user_exec_list[i].offset,
+					   &exec2_list[i].offset,
+					   sizeof(user_exec_list[i].offset)))
 				break;
-			}
 		}
 	}
 
-	drm_free_large(exec_list);
-	drm_free_large(exec2_list);
-	return ret;
+	kvfree(exec_list);
+	kvfree(exec2_list);
+	return err;
 }
 
 int
 i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		     struct drm_file *file)
 {
+	const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
 	struct drm_i915_gem_execbuffer2 *args = data;
-	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-	int ret;
+	struct drm_i915_gem_exec_object2 *exec2_list;
+	int err;
 
-	if (args->buffer_count < 1 ||
-	    args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
+	if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
 		DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
 		return -EINVAL;
 	}
 
-	exec2_list = drm_malloc_gfp(args->buffer_count,
-				    sizeof(*exec2_list),
-				    GFP_TEMPORARY);
+	if (!i915_gem_check_execbuffer(args))
+		return -EINVAL;
+
+	/* Allocate an extra slot for use by the command parser */
+	exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+				    __GFP_NOWARN | GFP_TEMPORARY);
 	if (exec2_list == NULL) {
 		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
 			  args->buffer_count);
 		return -ENOMEM;
 	}
-	ret = copy_from_user(exec2_list,
-			     u64_to_user_ptr(args->buffers_ptr),
-			     sizeof(*exec2_list) * args->buffer_count);
-	if (ret != 0) {
-		DRM_DEBUG("copy %d exec entries failed %d\n",
-			  args->buffer_count, ret);
-		drm_free_large(exec2_list);
+	if (copy_from_user(exec2_list,
+			   u64_to_user_ptr(args->buffers_ptr),
+			   sizeof(*exec2_list) * args->buffer_count)) {
+		DRM_DEBUG("copy %d exec entries failed\n", args->buffer_count);
+		kvfree(exec2_list);
 		return -EFAULT;
 	}
 
-	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
-	if (!ret) {
-		/* Copy the new buffer offsets back to the user's exec list. */
+	err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
+
+	/*
+	 * Now that we have begun execution of the batchbuffer, we ignore
+	 * any new error after this point. Also given that we have already
+	 * updated the associated relocations, we try to write out the current
+	 * object locations irrespective of any error.
+	 */
+	if (args->flags & __EXEC_HAS_RELOC) {
 		struct drm_i915_gem_exec_object2 __user *user_exec_list =
-				   u64_to_user_ptr(args->buffers_ptr);
-		int i;
+			u64_to_user_ptr(args->buffers_ptr);
+		unsigned int i;
 
+		/* Copy the new buffer offsets back to the user's exec list. */
+		user_access_begin();
 		for (i = 0; i < args->buffer_count; i++) {
+			if (!(exec2_list[i].offset & UPDATE))
+				continue;
+
 			exec2_list[i].offset =
-				gen8_canonical_addr(exec2_list[i].offset);
-			ret = __copy_to_user(&user_exec_list[i].offset,
-					     &exec2_list[i].offset,
-					     sizeof(user_exec_list[i].offset));
-			if (ret) {
-				ret = -EFAULT;
-				DRM_DEBUG("failed to copy %d exec entries "
-					  "back to user\n",
-					  args->buffer_count);
-				break;
-			}
+				gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
+			unsafe_put_user(exec2_list[i].offset,
+					&user_exec_list[i].offset,
+					end_user);
 		}
+end_user:
+		user_access_end();
 	}
 
-	drm_free_large(exec2_list);
-	return ret;
+	args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
+	kvfree(exec2_list);
+	return err;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index f1989b8792dd..61fc7e90a7da 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -168,13 +168,11 @@ int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 	if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
 		return 3;
 
-#ifdef CONFIG_INTEL_IOMMU
 	/* Disable ppgtt on SNB if VT-d is on. */
-	if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
+	if (IS_GEN6(dev_priv) && intel_vtd_active()) {
 		DRM_INFO("Disabling PPGTT because VT-d is on\n");
 		return 0;
 	}
-#endif
 
 	/* Early VLV doesn't have this */
 	if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
@@ -1886,7 +1884,7 @@ static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
 	 * called on driver load and after a GPU reset, so you can place
 	 * workarounds here even if they get overwritten by GPU reset.
 	 */
-	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk */
+	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl */
 	if (IS_BROADWELL(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
 	else if (IS_CHERRYVIEW(dev_priv))
@@ -1992,14 +1990,10 @@ void i915_ppgtt_release(struct kref *kref)
  */
 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
 {
-#ifdef CONFIG_INTEL_IOMMU
 	/* Query intel_iommu to see if we need the workaround. Presumably that
 	 * was loaded first.
 	 */
-	if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
-		return true;
-#endif
-	return false;
+	return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active();
 }
 
 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
@@ -2678,14 +2672,14 @@ static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
 {
 	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
 	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
-	return snb_gmch_ctl << 25; /* 32 MB units */
+	return (size_t)snb_gmch_ctl << 25; /* 32 MB units */
 }
 
 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
 {
 	bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
 	bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
-	return bdw_gmch_ctl << 25; /* 32 MB units */
+	return (size_t)bdw_gmch_ctl << 25; /* 32 MB units */
 }
 
 static size_t chv_get_stolen_size(u16 gmch_ctrl)
@@ -2699,11 +2693,11 @@ static size_t chv_get_stolen_size(u16 gmch_ctrl)
 	 * 0x17 to 0x1d: 4MB increments start at 36MB
 	 */
 	if (gmch_ctrl < 0x11)
-		return gmch_ctrl << 25;
+		return (size_t)gmch_ctrl << 25;
 	else if (gmch_ctrl < 0x17)
-		return (gmch_ctrl - 0x11 + 2) << 22;
+		return (size_t)(gmch_ctrl - 0x11 + 2) << 22;
 	else
-		return (gmch_ctrl - 0x17 + 9) << 22;
+		return (size_t)(gmch_ctrl - 0x17 + 9) << 22;
 }
 
 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
@@ -2712,10 +2706,10 @@ static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
 	gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
 
 	if (gen9_gmch_ctl < 0xf0)
-		return gen9_gmch_ctl << 25; /* 32 MB units */
+		return (size_t)gen9_gmch_ctl << 25; /* 32 MB units */
 	else
 		/* 4MB increments starting at 0xf0 for 4MB */
-		return (gen9_gmch_ctl - 0xf0 + 1) << 22;
+		return (size_t)(gen9_gmch_ctl - 0xf0 + 1) << 22;
 }
 
 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
@@ -2842,13 +2836,17 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	unsigned int size;
 	u16 snb_gmch_ctl;
+	int err;
 
 	/* TODO: We're not aware of mappable constraints on gen8 yet */
 	ggtt->mappable_base = pci_resource_start(pdev, 2);
 	ggtt->mappable_end = pci_resource_len(pdev, 2);
 
-	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
-		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
+	if (!err)
+		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
+	if (err)
+		DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
 
 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
@@ -2899,6 +2897,7 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	unsigned int size;
 	u16 snb_gmch_ctl;
+	int err;
 
 	ggtt->mappable_base = pci_resource_start(pdev, 2);
 	ggtt->mappable_end = pci_resource_len(pdev, 2);
@@ -2911,8 +2910,11 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 		return -ENXIO;
 	}
 
-	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
-		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40));
+	if (!err)
+		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
+	if (err)
+		DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
 	ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
@@ -3031,10 +3033,8 @@ int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
 		 ggtt->base.total >> 20);
 	DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
 	DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20);
-#ifdef CONFIG_INTEL_IOMMU
-	if (intel_iommu_gfx_mapped)
+	if (intel_vtd_active())
 		DRM_INFO("VT-d active for gfx access\n");
-#endif
 
 	return 0;
 }
@@ -3095,13 +3095,17 @@ int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
 
 void i915_ggtt_enable_guc(struct drm_i915_private *i915)
 {
+	GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate);
+
 	i915->ggtt.invalidate = guc_ggtt_invalidate;
 }
 
 void i915_ggtt_disable_guc(struct drm_i915_private *i915)
 {
-	if (i915->ggtt.invalidate == guc_ggtt_invalidate)
-		i915->ggtt.invalidate = gen6_ggtt_invalidate;
+	/* We should only be called after i915_ggtt_enable_guc() */
+	GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate);
+
+	i915->ggtt.invalidate = gen6_ggtt_invalidate;
 }
 
 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
@@ -3210,7 +3214,7 @@ intel_rotate_pages(struct intel_rotation_info *rot_info,
 	int ret = -ENOMEM;
 
 	/* Allocate a temporary list of source pages for random access. */
-	page_addr_list = drm_malloc_gfp(n_pages,
+	page_addr_list = kvmalloc_array(n_pages,
 					sizeof(dma_addr_t),
 					GFP_TEMPORARY);
 	if (!page_addr_list)
@@ -3243,14 +3247,14 @@ intel_rotate_pages(struct intel_rotation_info *rot_info,
 	DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
 		      obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
 
-	drm_free_large(page_addr_list);
+	kvfree(page_addr_list);
 
 	return st;
 
 err_sg_alloc:
 	kfree(st);
 err_st_alloc:
-	drm_free_large(page_addr_list);
+	kvfree(page_addr_list);
 
 	DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
 		      obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
@@ -3398,6 +3402,9 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm,
 	if (err != -ENOSPC)
 		return err;
 
+	if (flags & PIN_NOEVICT)
+		return -ENOSPC;
+
 	err = i915_gem_evict_for_node(vm, node, flags);
 	if (err == 0)
 		err = drm_mm_reserve_node(&vm->mm, node);
@@ -3512,6 +3519,9 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
 	if (err != -ENOSPC)
 		return err;
 
+	if (flags & PIN_NOEVICT)
+		return -ENOSPC;
+
 	/* No free space, pick a slot at random.
 	 *
 	 * There is a pathological case here using a GTT shared between
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index fb15684c1d83..1b2a56c3e5d3 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -255,6 +255,7 @@ struct i915_address_space {
 	struct drm_i915_file_private *file;
 	struct list_head global_link;
 	u64 total;		/* size addr space maps (ex. 2GB for ggtt) */
+	u64 reserved;		/* size addr space reserved */
 
 	bool closed;
 
@@ -588,6 +589,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
 #define PIN_MAPPABLE		BIT(1)
 #define PIN_ZONE_4G		BIT(2)
 #define PIN_NONFAULT		BIT(3)
+#define PIN_NOEVICT		BIT(4)
 
 #define PIN_MBZ			BIT(5) /* I915_VMA_PIN_OVERFLOW */
 #define PIN_GLOBAL		BIT(6) /* I915_VMA_GLOBAL_BIND */
diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
index fc950abbe400..568bf83af1f5 100644
--- a/drivers/gpu/drm/i915/i915_gem_internal.c
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -188,9 +188,11 @@ i915_gem_object_create_internal(struct drm_i915_private *i915,
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
 	i915_gem_object_init(obj, &i915_gem_object_internal_ops);
 
-	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	obj->cache_coherent = i915_gem_object_is_coherent(obj);
+	obj->cache_dirty = !obj->cache_coherent;
 
 	return obj;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 174cf923c236..5b19a4916a4d 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -37,8 +37,8 @@
 
 struct drm_i915_gem_object_ops {
 	unsigned int flags;
-#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1
-#define I915_GEM_OBJECT_IS_SHRINKABLE   0x2
+#define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0)
+#define I915_GEM_OBJECT_IS_SHRINKABLE   BIT(1)
 
 	/* Interface between the GEM object and its backing storage.
 	 * get_pages() is called once prior to the use of the associated set
@@ -68,9 +68,25 @@ struct drm_i915_gem_object {
 
 	const struct drm_i915_gem_object_ops *ops;
 
-	/** List of VMAs backed by this object */
+	/**
+	 * @vma_list: List of VMAs backed by this object
+	 *
+	 * The VMA on this list are ordered by type, all GGTT vma are placed
+	 * at the head and all ppGTT vma are placed at the tail. The different
+	 * types of GGTT vma are unordered between themselves, use the
+	 * @vma_tree (which has a defined order between all VMA) to find an
+	 * exact match.
+	 */
 	struct list_head vma_list;
+	/**
+	 * @vma_tree: Ordered tree of VMAs backed by this object
+	 *
+	 * All VMA created for this object are placed in the @vma_tree for
+	 * fast retrieval via a binary search in i915_vma_instance().
+	 * They are also added to @vma_list for easy iteration.
+	 */
 	struct rb_root vma_tree;
+	struct i915_vma *vma_hashed;
 
 	/** Stolen memory for this object, instead of being backed by shmem. */
 	struct drm_mm_node *stolen;
@@ -85,9 +101,6 @@ struct drm_i915_gem_object {
 	 */
 	struct list_head userfault_link;
 
-	/** Used in execbuf to temporarily hold a ref */
-	struct list_head obj_exec_link;
-
 	struct list_head batch_pool_link;
 	I915_SELFTEST_DECLARE(struct list_head st_link);
 
@@ -106,6 +119,7 @@ struct drm_i915_gem_object {
 	unsigned long gt_ro:1;
 	unsigned int cache_level:3;
 	unsigned int cache_dirty:1;
+	unsigned int cache_coherent:1;
 
 	atomic_t frontbuffer_bits;
 	unsigned int frontbuffer_ggtt_origin; /* write once */
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index a74d0ac737cb..8c59c79cbd8b 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -61,8 +61,8 @@ static bool i915_fence_enable_signaling(struct dma_fence *fence)
 	if (i915_fence_signaled(fence))
 		return false;
 
-	intel_engine_enable_signaling(to_request(fence));
-	return true;
+	intel_engine_enable_signaling(to_request(fence), true);
+	return !i915_fence_signaled(fence);
 }
 
 static signed long i915_fence_wait(struct dma_fence *fence,
@@ -159,7 +159,7 @@ i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
 {
 	struct i915_dependency *dep, *next;
 
-	GEM_BUG_ON(!RB_EMPTY_NODE(&pt->node));
+	GEM_BUG_ON(!list_empty(&pt->link));
 
 	/* Everyone we depended upon (the fences we wait to be signaled)
 	 * should retire before us and remove themselves from our list.
@@ -185,7 +185,7 @@ i915_priotree_init(struct i915_priotree *pt)
 {
 	INIT_LIST_HEAD(&pt->signalers_list);
 	INIT_LIST_HEAD(&pt->waiters_list);
-	RB_CLEAR_NODE(&pt->node);
+	INIT_LIST_HEAD(&pt->link);
 	pt->priority = INT_MIN;
 }
 
@@ -214,12 +214,12 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
 		}
 
 		/* Finally reset hw state */
-		tl->seqno = seqno;
 		intel_engine_init_global_seqno(engine, seqno);
+		tl->seqno = seqno;
 
 		list_for_each_entry(timeline, &i915->gt.timelines, link)
-			memset(timeline->engine[id].sync_seqno, 0,
-			       sizeof(timeline->engine[id].sync_seqno));
+			memset(timeline->engine[id].global_sync, 0,
+			       sizeof(timeline->engine[id].global_sync));
 	}
 
 	return 0;
@@ -271,6 +271,48 @@ void i915_gem_retire_noop(struct i915_gem_active *active,
 	/* Space left intentionally blank */
 }
 
+static void advance_ring(struct drm_i915_gem_request *request)
+{
+	unsigned int tail;
+
+	/* We know the GPU must have read the request to have
+	 * sent us the seqno + interrupt, so use the position
+	 * of tail of the request to update the last known position
+	 * of the GPU head.
+	 *
+	 * Note this requires that we are always called in request
+	 * completion order.
+	 */
+	if (list_is_last(&request->ring_link, &request->ring->request_list)) {
+		/* We may race here with execlists resubmitting this request
+		 * as we retire it. The resubmission will move the ring->tail
+		 * forwards (to request->wa_tail). We either read the
+		 * current value that was written to hw, or the value that
+		 * is just about to be. Either works, if we miss the last two
+		 * noops - they are safe to be replayed on a reset.
+		 */
+		tail = READ_ONCE(request->ring->tail);
+	} else {
+		tail = request->postfix;
+	}
+	list_del(&request->ring_link);
+
+	request->ring->head = tail;
+}
+
+static void free_capture_list(struct drm_i915_gem_request *request)
+{
+	struct i915_gem_capture_list *capture;
+
+	capture = request->capture_list;
+	while (capture) {
+		struct i915_gem_capture_list *next = capture->next;
+
+		kfree(capture);
+		capture = next;
+	}
+}
+
 static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
@@ -287,16 +329,6 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 	list_del_init(&request->link);
 	spin_unlock_irq(&engine->timeline->lock);
 
-	/* We know the GPU must have read the request to have
-	 * sent us the seqno + interrupt, so use the position
-	 * of tail of the request to update the last known position
-	 * of the GPU head.
-	 *
-	 * Note this requires that we are always called in request
-	 * completion order.
-	 */
-	list_del(&request->ring_link);
-	request->ring->head = request->postfix;
 	if (!--request->i915->gt.active_requests) {
 		GEM_BUG_ON(!request->i915->gt.awake);
 		mod_delayed_work(request->i915->wq,
@@ -304,6 +336,9 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 				 msecs_to_jiffies(100));
 	}
 	unreserve_seqno(request->engine);
+	advance_ring(request);
+
+	free_capture_list(request);
 
 	/* Walk through the active list, calling retire on each. This allows
 	 * objects to track their GPU activity and mark themselves as idle
@@ -402,7 +437,7 @@ void __i915_gem_request_submit(struct drm_i915_gem_request *request)
 	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
 	request->global_seqno = seqno;
 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
-		intel_engine_enable_signaling(request);
+		intel_engine_enable_signaling(request, false);
 	spin_unlock(&request->lock);
 
 	engine->emit_breadcrumb(request,
@@ -503,9 +538,6 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
  *
  * @engine: engine that we wish to issue the request on.
  * @ctx: context that the request will be associated with.
- *       This can be NULL if the request is not directly related to
- *       any specific user context, in which case this function will
- *       choose an appropriate context to use.
  *
  * Returns a pointer to the allocated request if successful,
  * or an error code if not.
@@ -516,6 +548,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 	struct drm_i915_gem_request *req;
+	struct intel_ring *ring;
 	int ret;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
@@ -530,9 +563,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	 * GGTT space, so do this first before we reserve a seqno for
 	 * ourselves.
 	 */
-	ret = engine->context_pin(engine, ctx);
-	if (ret)
-		return ERR_PTR(ret);
+	ring = engine->context_pin(engine, ctx);
+	if (IS_ERR(ring))
+		return ERR_CAST(ring);
+	GEM_BUG_ON(!ring);
 
 	ret = reserve_seqno(engine);
 	if (ret)
@@ -598,11 +632,13 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	req->i915 = dev_priv;
 	req->engine = engine;
 	req->ctx = ctx;
+	req->ring = ring;
 
 	/* No zalloc, must clear what we need by hand */
 	req->global_seqno = 0;
 	req->file_priv = NULL;
 	req->batch = NULL;
+	req->capture_list = NULL;
 
 	/*
 	 * Reserve space in the ring buffer for all the commands required to
@@ -647,10 +683,10 @@ static int
 i915_gem_request_await_request(struct drm_i915_gem_request *to,
 			       struct drm_i915_gem_request *from)
 {
-	u32 seqno;
 	int ret;
 
 	GEM_BUG_ON(to == from);
+	GEM_BUG_ON(to->timeline == from->timeline);
 
 	if (i915_gem_request_completed(from))
 		return 0;
@@ -663,9 +699,6 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 			return ret;
 	}
 
-	if (to->timeline == from->timeline)
-		return 0;
-
 	if (to->engine == from->engine) {
 		ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
 						       &from->submit,
@@ -673,56 +706,41 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 		return ret < 0 ? ret : 0;
 	}
 
-	seqno = i915_gem_request_global_seqno(from);
-	if (!seqno) {
-		ret = i915_sw_fence_await_dma_fence(&to->submit,
-						    &from->fence, 0,
-						    GFP_KERNEL);
-		return ret < 0 ? ret : 0;
-	}
+	if (to->engine->semaphore.sync_to) {
+		u32 seqno;
 
-	if (seqno <= to->timeline->sync_seqno[from->engine->id])
-		return 0;
+		GEM_BUG_ON(!from->engine->semaphore.signal);
 
-	trace_i915_gem_ring_sync_to(to, from);
-	if (!i915.semaphores) {
-		if (!i915_spin_request(from, TASK_INTERRUPTIBLE, 2)) {
-			ret = i915_sw_fence_await_dma_fence(&to->submit,
-							    &from->fence, 0,
-							    GFP_KERNEL);
-			if (ret < 0)
-				return ret;
-		}
-	} else {
+		seqno = i915_gem_request_global_seqno(from);
+		if (!seqno)
+			goto await_dma_fence;
+
+		if (seqno <= to->timeline->global_sync[from->engine->id])
+			return 0;
+
+		trace_i915_gem_ring_sync_to(to, from);
 		ret = to->engine->semaphore.sync_to(to, from);
 		if (ret)
 			return ret;
+
+		to->timeline->global_sync[from->engine->id] = seqno;
+		return 0;
 	}
 
-	to->timeline->sync_seqno[from->engine->id] = seqno;
-	return 0;
+await_dma_fence:
+	ret = i915_sw_fence_await_dma_fence(&to->submit,
+					    &from->fence, 0,
+					    GFP_KERNEL);
+	return ret < 0 ? ret : 0;
 }
 
 int
 i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
 				 struct dma_fence *fence)
 {
-	struct dma_fence_array *array;
+	struct dma_fence **child = &fence;
+	unsigned int nchild = 1;
 	int ret;
-	int i;
-
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
-		return 0;
-
-	if (dma_fence_is_i915(fence))
-		return i915_gem_request_await_request(req, to_request(fence));
-
-	if (!dma_fence_is_array(fence)) {
-		ret = i915_sw_fence_await_dma_fence(&req->submit,
-						    fence, I915_FENCE_TIMEOUT,
-						    GFP_KERNEL);
-		return ret < 0 ? ret : 0;
-	}
 
 	/* Note that if the fence-array was created in signal-on-any mode,
 	 * we should *not* decompose it into its individual fences. However,
@@ -731,21 +749,46 @@ i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
 	 * amdgpu and we should not see any incoming fence-array from
 	 * sync-file being in signal-on-any mode.
 	 */
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
 
-	array = to_dma_fence_array(fence);
-	for (i = 0; i < array->num_fences; i++) {
-		struct dma_fence *child = array->fences[i];
+		child = array->fences;
+		nchild = array->num_fences;
+		GEM_BUG_ON(!nchild);
+	}
 
-		if (dma_fence_is_i915(child))
+	do {
+		fence = *child++;
+		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+			continue;
+
+		/*
+		 * Requests on the same timeline are explicitly ordered, along
+		 * with their dependencies, by i915_add_request() which ensures
+		 * that requests are submitted in-order through each ring.
+		 */
+		if (fence->context == req->fence.context)
+			continue;
+
+		/* Squash repeated waits to the same timelines */
+		if (fence->context != req->i915->mm.unordered_timeline &&
+		    intel_timeline_sync_is_later(req->timeline, fence))
+			continue;
+
+		if (dma_fence_is_i915(fence))
 			ret = i915_gem_request_await_request(req,
-							     to_request(child));
+							     to_request(fence));
 		else
-			ret = i915_sw_fence_await_dma_fence(&req->submit,
-							    child, I915_FENCE_TIMEOUT,
+			ret = i915_sw_fence_await_dma_fence(&req->submit, fence,
+							    I915_FENCE_TIMEOUT,
 							    GFP_KERNEL);
 		if (ret < 0)
 			return ret;
-	}
+
+		/* Record the latest fence used against each timeline */
+		if (fence->context != req->i915->mm.unordered_timeline)
+			intel_timeline_sync_set(req->timeline, fence);
+	} while (--nchild);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
index a4a920c4c454..7579b9702c22 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.h
+++ b/drivers/gpu/drm/i915/i915_gem_request.h
@@ -67,12 +67,18 @@ struct i915_dependency {
 struct i915_priotree {
 	struct list_head signalers_list; /* those before us, we depend upon */
 	struct list_head waiters_list; /* those after us, they depend upon us */
-	struct rb_node node;
+	struct list_head link;
 	int priority;
 #define I915_PRIORITY_MAX 1024
+#define I915_PRIORITY_NORMAL 0
 #define I915_PRIORITY_MIN (-I915_PRIORITY_MAX)
 };
 
+struct i915_gem_capture_list {
+	struct i915_gem_capture_list *next;
+	struct i915_vma *vma;
+};
+
 /**
  * Request queue structure.
  *
@@ -167,6 +173,12 @@ struct drm_i915_gem_request {
 	 * error state dump only).
 	 */
 	struct i915_vma *batch;
+	/** Additional buffers requested by userspace to be captured upon
+	 * a GPU hang. The vma/obj on this list are protected by their
+	 * active reference - all objects on this list must also be
+	 * on the active_list (of their final request).
+	 */
+	struct i915_gem_capture_list *capture_list;
 	struct list_head active_list;
 
 	/** Time at which this request was emitted, in jiffies. */
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 57d9f7f4ef15..1032f98add11 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -35,39 +35,52 @@
 #include "i915_drv.h"
 #include "i915_trace.h"
 
-static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
+static bool shrinker_lock(struct drm_i915_private *dev_priv, bool *unlock)
 {
-	switch (mutex_trylock_recursive(&dev->struct_mutex)) {
-	case MUTEX_TRYLOCK_FAILED:
-		return false;
-
-	case MUTEX_TRYLOCK_SUCCESS:
-		*unlock = true;
-		return true;
-
+	switch (mutex_trylock_recursive(&dev_priv->drm.struct_mutex)) {
 	case MUTEX_TRYLOCK_RECURSIVE:
 		*unlock = false;
 		return true;
+
+	case MUTEX_TRYLOCK_FAILED:
+		do {
+			cpu_relax();
+			if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
+	case MUTEX_TRYLOCK_SUCCESS:
+				*unlock = true;
+				return true;
+			}
+		} while (!need_resched());
+
+		return false;
 	}
 
 	BUG();
 }
 
-static void i915_gem_shrinker_unlock(struct drm_device *dev, bool unlock)
+static void shrinker_unlock(struct drm_i915_private *dev_priv, bool unlock)
 {
 	if (!unlock)
 		return;
 
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 }
 
 static bool any_vma_pinned(struct drm_i915_gem_object *obj)
 {
 	struct i915_vma *vma;
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link)
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		/* Only GGTT vma may be permanently pinned, and are always
+		 * at the start of the list. We can stop hunting as soon
+		 * as we see a ppGTT vma.
+		 */
+		if (!i915_vma_is_ggtt(vma))
+			break;
+
 		if (i915_vma_is_pinned(vma))
 			return true;
+	}
 
 	return false;
 }
@@ -153,7 +166,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 	unsigned long count = 0;
 	bool unlock;
 
-	if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock))
+	if (!shrinker_lock(dev_priv, &unlock))
 		return 0;
 
 	trace_i915_gem_shrink(dev_priv, target, flags);
@@ -241,7 +254,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
 	i915_gem_retire_requests(dev_priv);
 
-	i915_gem_shrinker_unlock(&dev_priv->drm, unlock);
+	shrinker_unlock(dev_priv, unlock);
 
 	return count;
 }
@@ -279,12 +292,11 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(shrinker, struct drm_i915_private, mm.shrinker);
-	struct drm_device *dev = &dev_priv->drm;
 	struct drm_i915_gem_object *obj;
 	unsigned long count;
 	bool unlock;
 
-	if (!i915_gem_shrinker_lock(dev, &unlock))
+	if (!shrinker_lock(dev_priv, &unlock))
 		return 0;
 
 	i915_gem_retire_requests(dev_priv);
@@ -299,7 +311,7 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 			count += obj->base.size >> PAGE_SHIFT;
 	}
 
-	i915_gem_shrinker_unlock(dev, unlock);
+	shrinker_unlock(dev_priv, unlock);
 
 	return count;
 }
@@ -309,11 +321,10 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(shrinker, struct drm_i915_private, mm.shrinker);
-	struct drm_device *dev = &dev_priv->drm;
 	unsigned long freed;
 	bool unlock;
 
-	if (!i915_gem_shrinker_lock(dev, &unlock))
+	if (!shrinker_lock(dev_priv, &unlock))
 		return SHRINK_STOP;
 
 	freed = i915_gem_shrink(dev_priv,
@@ -326,27 +337,30 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 					 sc->nr_to_scan - freed,
 					 I915_SHRINK_BOUND |
 					 I915_SHRINK_UNBOUND);
+	if (freed < sc->nr_to_scan && current_is_kswapd()) {
+		intel_runtime_pm_get(dev_priv);
+		freed += i915_gem_shrink(dev_priv,
+					 sc->nr_to_scan - freed,
+					 I915_SHRINK_ACTIVE |
+					 I915_SHRINK_BOUND |
+					 I915_SHRINK_UNBOUND);
+		intel_runtime_pm_put(dev_priv);
+	}
 
-	i915_gem_shrinker_unlock(dev, unlock);
+	shrinker_unlock(dev_priv, unlock);
 
 	return freed;
 }
 
-struct shrinker_lock_uninterruptible {
-	bool was_interruptible;
-	bool unlock;
-};
-
 static bool
-i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
-				       struct shrinker_lock_uninterruptible *slu,
-				       int timeout_ms)
+shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, bool *unlock,
+			      int timeout_ms)
 {
 	unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
 
 	do {
 		if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
-		    i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock))
+		    shrinker_lock(dev_priv, unlock))
 			break;
 
 		schedule_timeout_killable(1);
@@ -359,29 +373,19 @@ i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
 		}
 	} while (1);
 
-	slu->was_interruptible = dev_priv->mm.interruptible;
-	dev_priv->mm.interruptible = false;
 	return true;
 }
 
-static void
-i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
-					 struct shrinker_lock_uninterruptible *slu)
-{
-	dev_priv->mm.interruptible = slu->was_interruptible;
-	i915_gem_shrinker_unlock(&dev_priv->drm, slu->unlock);
-}
-
 static int
 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(nb, struct drm_i915_private, mm.oom_notifier);
-	struct shrinker_lock_uninterruptible slu;
 	struct drm_i915_gem_object *obj;
 	unsigned long unevictable, bound, unbound, freed_pages;
+	bool unlock;
 
-	if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
+	if (!shrinker_lock_uninterruptible(dev_priv, &unlock, 5000))
 		return NOTIFY_DONE;
 
 	freed_pages = i915_gem_shrink_all(dev_priv);
@@ -410,7 +414,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
 			bound += obj->base.size >> PAGE_SHIFT;
 	}
 
-	i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
+	shrinker_unlock(dev_priv, unlock);
 
 	if (freed_pages || unbound || bound)
 		pr_info("Purging GPU memory, %lu pages freed, "
@@ -430,12 +434,12 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
 {
 	struct drm_i915_private *dev_priv =
 		container_of(nb, struct drm_i915_private, mm.vmap_notifier);
-	struct shrinker_lock_uninterruptible slu;
 	struct i915_vma *vma, *next;
 	unsigned long freed_pages = 0;
+	bool unlock;
 	int ret;
 
-	if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
+	if (!shrinker_lock_uninterruptible(dev_priv, &unlock, 5000))
 		return NOTIFY_DONE;
 
 	/* Force everything onto the inactive lists */
@@ -460,7 +464,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
 	}
 
 out:
-	i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
+	shrinker_unlock(dev_priv, unlock);
 
 	*(unsigned long *)ptr += freed_pages;
 	return NOTIFY_DONE;
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index f3abdc27c5dd..a817b3e0b17e 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -414,12 +414,10 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 		return 0;
 	}
 
-#ifdef CONFIG_INTEL_IOMMU
-	if (intel_iommu_gfx_mapped && INTEL_GEN(dev_priv) < 8) {
+	if (intel_vtd_active() && INTEL_GEN(dev_priv) < 8) {
 		DRM_INFO("DMAR active, disabling use of stolen memory\n");
 		return 0;
 	}
-#endif
 
 	if (ggtt->stolen_size == 0)
 		return 0;
@@ -592,6 +590,7 @@ _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 	obj->stolen = stolen;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 	obj->cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	obj->cache_coherent = true; /* assumptions! more like cache_oblivious */
 
 	if (i915_gem_object_pin_pages(obj))
 		goto cleanup;
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c
index b596ca7ee058..c597ce277a04 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.c
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.c
@@ -23,6 +23,32 @@
  */
 
 #include "i915_drv.h"
+#include "i915_syncmap.h"
+
+static void __intel_timeline_init(struct intel_timeline *tl,
+				  struct i915_gem_timeline *parent,
+				  u64 context,
+				  struct lock_class_key *lockclass,
+				  const char *lockname)
+{
+	tl->fence_context = context;
+	tl->common = parent;
+#ifdef CONFIG_DEBUG_SPINLOCK
+	__raw_spin_lock_init(&tl->lock.rlock, lockname, lockclass);
+#else
+	spin_lock_init(&tl->lock);
+#endif
+	init_request_active(&tl->last_request, NULL);
+	INIT_LIST_HEAD(&tl->requests);
+	i915_syncmap_init(&tl->sync);
+}
+
+static void __intel_timeline_fini(struct intel_timeline *tl)
+{
+	GEM_BUG_ON(!list_empty(&tl->requests));
+
+	i915_syncmap_free(&tl->sync);
+}
 
 static int __i915_gem_timeline_init(struct drm_i915_private *i915,
 				    struct i915_gem_timeline *timeline,
@@ -35,6 +61,14 @@ static int __i915_gem_timeline_init(struct drm_i915_private *i915,
 
 	lockdep_assert_held(&i915->drm.struct_mutex);
 
+	/*
+	 * Ideally we want a set of engines on a single leaf as we expect
+	 * to mostly be tracking synchronisation between engines. It is not
+	 * a huge issue if this is not the case, but we may want to mitigate
+	 * any page crossing penalties if they become an issue.
+	 */
+	BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
+
 	timeline->i915 = i915;
 	timeline->name = kstrdup(name ?: "[kernel]", GFP_KERNEL);
 	if (!timeline->name)
@@ -44,19 +78,10 @@ static int __i915_gem_timeline_init(struct drm_i915_private *i915,
 
 	/* Called during early_init before we know how many engines there are */
 	fences = dma_fence_context_alloc(ARRAY_SIZE(timeline->engine));
-	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) {
-		struct intel_timeline *tl = &timeline->engine[i];
-
-		tl->fence_context = fences++;
-		tl->common = timeline;
-#ifdef CONFIG_DEBUG_SPINLOCK
-		__raw_spin_lock_init(&tl->lock.rlock, lockname, lockclass);
-#else
-		spin_lock_init(&tl->lock);
-#endif
-		init_request_active(&tl->last_request, NULL);
-		INIT_LIST_HEAD(&tl->requests);
-	}
+	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++)
+		__intel_timeline_init(&timeline->engine[i],
+				      timeline, fences++,
+				      lockclass, lockname);
 
 	return 0;
 }
@@ -81,18 +106,52 @@ int i915_gem_timeline_init__global(struct drm_i915_private *i915)
 					&class, "&global_timeline->lock");
 }
 
+/**
+ * i915_gem_timelines_mark_idle -- called when the driver idles
+ * @i915 - the drm_i915_private device
+ *
+ * When the driver is completely idle, we know that all of our sync points
+ * have been signaled and our tracking is then entirely redundant. Any request
+ * to wait upon an older sync point will be completed instantly as we know
+ * the fence is signaled and therefore we will not even look them up in the
+ * sync point map.
+ */
+void i915_gem_timelines_mark_idle(struct drm_i915_private *i915)
+{
+	struct i915_gem_timeline *timeline;
+	int i;
+
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	list_for_each_entry(timeline, &i915->gt.timelines, link) {
+		for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) {
+			struct intel_timeline *tl = &timeline->engine[i];
+
+			/*
+			 * All known fences are completed so we can scrap
+			 * the current sync point tracking and start afresh,
+			 * any attempt to wait upon a previous sync point
+			 * will be skipped as the fence was signaled.
+			 */
+			i915_syncmap_free(&tl->sync);
+		}
+	}
+}
+
 void i915_gem_timeline_fini(struct i915_gem_timeline *timeline)
 {
 	int i;
 
 	lockdep_assert_held(&timeline->i915->drm.struct_mutex);
 
-	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) {
-		struct intel_timeline *tl = &timeline->engine[i];
-
-		GEM_BUG_ON(!list_empty(&tl->requests));
-	}
+	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++)
+		__intel_timeline_fini(&timeline->engine[i]);
 
 	list_del(&timeline->link);
 	kfree(timeline->name);
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/mock_timeline.c"
+#include "selftests/i915_gem_timeline.c"
+#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
index 6c53e14cab2a..bfb5eb94c64d 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.h
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.h
@@ -27,7 +27,9 @@
 
 #include <linux/list.h>
 
+#include "i915_utils.h"
 #include "i915_gem_request.h"
+#include "i915_syncmap.h"
 
 struct i915_gem_timeline;
 
@@ -55,7 +57,25 @@ struct intel_timeline {
 	 * struct_mutex.
 	 */
 	struct i915_gem_active last_request;
-	u32 sync_seqno[I915_NUM_ENGINES];
+
+	/**
+	 * We track the most recent seqno that we wait on in every context so
+	 * that we only have to emit a new await and dependency on a more
+	 * recent sync point. As the contexts may be executed out-of-order, we
+	 * have to track each individually and can not rely on an absolute
+	 * global_seqno. When we know that all tracked fences are completed
+	 * (i.e. when the driver is idle), we know that the syncmap is
+	 * redundant and we can discard it without loss of generality.
+	 */
+	struct i915_syncmap *sync;
+	/**
+	 * Separately to the inter-context seqno map above, we track the last
+	 * barrier (e.g. semaphore wait) to the global engine timelines. Note
+	 * that this tracks global_seqno rather than the context.seqno, and
+	 * so it is subject to the limitations of hw wraparound and that we
+	 * may need to revoke global_seqno (on pre-emption).
+	 */
+	u32 global_sync[I915_NUM_ENGINES];
 
 	struct i915_gem_timeline *common;
 };
@@ -73,6 +93,31 @@ int i915_gem_timeline_init(struct drm_i915_private *i915,
 			   struct i915_gem_timeline *tl,
 			   const char *name);
 int i915_gem_timeline_init__global(struct drm_i915_private *i915);
+void i915_gem_timelines_mark_idle(struct drm_i915_private *i915);
 void i915_gem_timeline_fini(struct i915_gem_timeline *tl);
 
+static inline int __intel_timeline_sync_set(struct intel_timeline *tl,
+					    u64 context, u32 seqno)
+{
+	return i915_syncmap_set(&tl->sync, context, seqno);
+}
+
+static inline int intel_timeline_sync_set(struct intel_timeline *tl,
+					  const struct dma_fence *fence)
+{
+	return __intel_timeline_sync_set(tl, fence->context, fence->seqno);
+}
+
+static inline bool __intel_timeline_sync_is_later(struct intel_timeline *tl,
+						  u64 context, u32 seqno)
+{
+	return i915_syncmap_is_later(&tl->sync, context, seqno);
+}
+
+static inline bool intel_timeline_sync_is_later(struct intel_timeline *tl,
+						const struct dma_fence *fence)
+{
+	return __intel_timeline_sync_is_later(tl, fence->context, fence->seqno);
+}
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 58ccf8b8ca1c..ccd09e8419f5 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -378,7 +378,7 @@ __i915_mm_struct_free(struct kref *kref)
 	mutex_unlock(&mm->i915->mm_lock);
 
 	INIT_WORK(&mm->work, __i915_mm_struct_free__worker);
-	schedule_work(&mm->work);
+	queue_work(mm->i915->mm.userptr_wq, &mm->work);
 }
 
 static void
@@ -507,7 +507,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 	ret = -ENOMEM;
 	pinned = 0;
 
-	pvec = drm_malloc_gfp(npages, sizeof(struct page *), GFP_TEMPORARY);
+	pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_TEMPORARY);
 	if (pvec != NULL) {
 		struct mm_struct *mm = obj->userptr.mm->mm;
 		unsigned int flags = 0;
@@ -555,7 +555,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 	mutex_unlock(&obj->mm.lock);
 
 	release_pages(pvec, pinned, 0);
-	drm_free_large(pvec);
+	kvfree(pvec);
 
 	i915_gem_object_put(obj);
 	put_task_struct(work->task);
@@ -598,7 +598,7 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj)
 	get_task_struct(work->task);
 
 	INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
-	schedule_work(&work->work);
+	queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work);
 
 	return ERR_PTR(-EAGAIN);
 }
@@ -642,7 +642,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 	pinned = 0;
 
 	if (mm == current->mm) {
-		pvec = drm_malloc_gfp(num_pages, sizeof(struct page *),
+		pvec = kvmalloc_array(num_pages, sizeof(struct page *),
 				      GFP_TEMPORARY |
 				      __GFP_NORETRY |
 				      __GFP_NOWARN);
@@ -669,7 +669,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 
 	if (IS_ERR(pages))
 		release_pages(pvec, pinned, 0);
-	drm_free_large(pvec);
+	kvfree(pvec);
 
 	return pages;
 }
@@ -802,9 +802,11 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file
 
 	drm_gem_private_object_init(dev, &obj->base, args->user_size);
 	i915_gem_object_init(obj, &i915_gem_userptr_ops);
-	obj->cache_level = I915_CACHE_LLC;
-	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	obj->cache_level = I915_CACHE_LLC;
+	obj->cache_coherent = i915_gem_object_is_coherent(obj);
+	obj->cache_dirty = !obj->cache_coherent;
 
 	obj->userptr.ptr = args->user_ptr;
 	obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY);
@@ -828,8 +830,20 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file
 	return 0;
 }
 
-void i915_gem_init_userptr(struct drm_i915_private *dev_priv)
+int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
 {
 	mutex_init(&dev_priv->mm_lock);
 	hash_init(dev_priv->mm_structs);
+
+	dev_priv->mm.userptr_wq =
+		alloc_workqueue("i915-userptr-acquire", WQ_HIGHPRI, 0);
+	if (!dev_priv->mm.userptr_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
+{
+	destroy_workqueue(dev_priv->mm.userptr_wq);
 }
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 8effc59f5cb5..e18f350bc364 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -712,6 +712,10 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			print_error_obj(m, dev_priv->engine[i], NULL, obj);
 		}
 
+		for (j = 0; j < ee->user_bo_count; j++)
+			print_error_obj(m, dev_priv->engine[i],
+					"user", ee->user_bo[j]);
+
 		if (ee->num_requests) {
 			err_printf(m, "%s --- %d requests\n",
 				   dev_priv->engine[i]->name,
@@ -825,11 +829,15 @@ void __i915_gpu_state_free(struct kref *error_ref)
 {
 	struct i915_gpu_state *error =
 		container_of(error_ref, typeof(*error), ref);
-	int i;
+	long i, j;
 
 	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
 		struct drm_i915_error_engine *ee = &error->engine[i];
 
+		for (j = 0; j < ee->user_bo_count; j++)
+			i915_error_object_free(ee->user_bo[j]);
+		kfree(ee->user_bo);
+
 		i915_error_object_free(ee->batchbuffer);
 		i915_error_object_free(ee->wa_batchbuffer);
 		i915_error_object_free(ee->ringbuffer);
@@ -1316,12 +1324,17 @@ static void engine_record_requests(struct intel_engine_cs *engine,
 static void error_record_engine_execlists(struct intel_engine_cs *engine,
 					  struct drm_i915_error_engine *ee)
 {
+	const struct execlist_port *port = engine->execlist_port;
 	unsigned int n;
 
-	for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
-		if (engine->execlist_port[n].request)
-			record_request(engine->execlist_port[n].request,
-				       &ee->execlist[n]);
+	for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++) {
+		struct drm_i915_gem_request *rq = port_request(&port[n]);
+
+		if (!rq)
+			break;
+
+		record_request(rq, &ee->execlist[n]);
+	}
 }
 
 static void record_context(struct drm_i915_error_context *e,
@@ -1346,6 +1359,35 @@ static void record_context(struct drm_i915_error_context *e,
 	e->active = ctx->active_count;
 }
 
+static void request_record_user_bo(struct drm_i915_gem_request *request,
+				   struct drm_i915_error_engine *ee)
+{
+	struct i915_gem_capture_list *c;
+	struct drm_i915_error_object **bo;
+	long count;
+
+	count = 0;
+	for (c = request->capture_list; c; c = c->next)
+		count++;
+
+	bo = NULL;
+	if (count)
+		bo = kcalloc(count, sizeof(*bo), GFP_ATOMIC);
+	if (!bo)
+		return;
+
+	count = 0;
+	for (c = request->capture_list; c; c = c->next) {
+		bo[count] = i915_error_object_create(request->i915, c->vma);
+		if (!bo[count])
+			break;
+		count++;
+	}
+
+	ee->user_bo = bo;
+	ee->user_bo_count = count;
+}
+
 static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 				  struct i915_gpu_state *error)
 {
@@ -1392,6 +1434,7 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 				ee->wa_batchbuffer =
 					i915_error_object_create(dev_priv,
 								 engine->scratch);
+			request_record_user_bo(request, ee);
 
 			ee->ctx =
 				i915_error_object_create(dev_priv,
@@ -1560,6 +1603,9 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 		error->done_reg = I915_READ(DONE_REG);
 	}
 
+	if (INTEL_GEN(dev_priv) >= 5)
+		error->ccid = I915_READ(CCID);
+
 	/* 3: Feature specific registers */
 	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
 		error->gam_ecochk = I915_READ(GAM_ECOCHK);
@@ -1567,9 +1613,6 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 	}
 
 	/* 4: Everything else */
-	if (HAS_HW_CONTEXTS(dev_priv))
-		error->ccid = I915_READ(CCID);
-
 	if (INTEL_GEN(dev_priv) >= 8) {
 		error->ier = I915_READ(GEN8_DE_MISC_IER);
 		for (i = 0; i < 4; i++)
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index ab5140ba108d..48a1e9349a2c 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -105,7 +105,7 @@ static int __reserve_doorbell(struct i915_guc_client *client)
 		end += offset;
 	}
 
-	id = find_next_zero_bit(client->guc->doorbell_bitmap, offset, end);
+	id = find_next_zero_bit(client->guc->doorbell_bitmap, end, offset);
 	if (id == end)
 		return -ENOSPC;
 
@@ -614,12 +614,6 @@ static void __i915_guc_submit(struct drm_i915_gem_request *rq)
 	b_ret = guc_ring_doorbell(client);
 
 	client->submissions[engine_id] += 1;
-	client->retcode = b_ret;
-	if (b_ret)
-		client->b_fail += 1;
-
-	guc->submissions[engine_id] += 1;
-	guc->last_seqno[engine_id] = rq->global_seqno;
 
 	spin_unlock_irqrestore(&client->wq_lock, flags);
 }
@@ -649,47 +643,68 @@ static void nested_enable_signaling(struct drm_i915_gem_request *rq)
 	trace_dma_fence_enable_signal(&rq->fence);
 
 	spin_lock_nested(&rq->lock, SINGLE_DEPTH_NESTING);
-	intel_engine_enable_signaling(rq);
+	intel_engine_enable_signaling(rq, true);
 	spin_unlock(&rq->lock);
 }
 
+static void port_assign(struct execlist_port *port,
+			struct drm_i915_gem_request *rq)
+{
+	GEM_BUG_ON(rq == port_request(port));
+
+	if (port_isset(port))
+		i915_gem_request_put(port_request(port));
+
+	port_set(port, i915_gem_request_get(rq));
+	nested_enable_signaling(rq);
+}
+
 static bool i915_guc_dequeue(struct intel_engine_cs *engine)
 {
 	struct execlist_port *port = engine->execlist_port;
-	struct drm_i915_gem_request *last = port[0].request;
+	struct drm_i915_gem_request *last = port_request(port);
 	struct rb_node *rb;
 	bool submit = false;
 
 	spin_lock_irq(&engine->timeline->lock);
 	rb = engine->execlist_first;
+	GEM_BUG_ON(rb_first(&engine->execlist_queue) != rb);
 	while (rb) {
-		struct drm_i915_gem_request *rq =
-			rb_entry(rb, typeof(*rq), priotree.node);
-
-		if (last && rq->ctx != last->ctx) {
-			if (port != engine->execlist_port)
-				break;
-
-			i915_gem_request_assign(&port->request, last);
-			nested_enable_signaling(last);
-			port++;
+		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
+		struct drm_i915_gem_request *rq, *rn;
+
+		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
+			if (last && rq->ctx != last->ctx) {
+				if (port != engine->execlist_port) {
+					__list_del_many(&p->requests,
+							&rq->priotree.link);
+					goto done;
+				}
+
+				if (submit)
+					port_assign(port, last);
+				port++;
+			}
+
+			INIT_LIST_HEAD(&rq->priotree.link);
+			rq->priotree.priority = INT_MAX;
+
+			i915_guc_submit(rq);
+			trace_i915_gem_request_in(rq, port_index(port, engine));
+			last = rq;
+			submit = true;
 		}
 
 		rb = rb_next(rb);
-		rb_erase(&rq->priotree.node, &engine->execlist_queue);
-		RB_CLEAR_NODE(&rq->priotree.node);
-		rq->priotree.priority = INT_MAX;
-
-		i915_guc_submit(rq);
-		trace_i915_gem_request_in(rq, port - engine->execlist_port);
-		last = rq;
-		submit = true;
-	}
-	if (submit) {
-		i915_gem_request_assign(&port->request, last);
-		nested_enable_signaling(last);
-		engine->execlist_first = rb;
+		rb_erase(&p->node, &engine->execlist_queue);
+		INIT_LIST_HEAD(&p->requests);
+		if (p->priority != I915_PRIORITY_NORMAL)
+			kmem_cache_free(engine->i915->priorities, p);
 	}
+done:
+	engine->execlist_first = rb;
+	if (submit)
+		port_assign(port, last);
 	spin_unlock_irq(&engine->timeline->lock);
 
 	return submit;
@@ -703,17 +718,19 @@ static void i915_guc_irq_handler(unsigned long data)
 	bool submit;
 
 	do {
-		rq = port[0].request;
+		rq = port_request(&port[0]);
 		while (rq && i915_gem_request_completed(rq)) {
 			trace_i915_gem_request_out(rq);
 			i915_gem_request_put(rq);
-			port[0].request = port[1].request;
-			port[1].request = NULL;
-			rq = port[0].request;
+
+			port[0] = port[1];
+			memset(&port[1], 0, sizeof(port[1]));
+
+			rq = port_request(&port[0]);
 		}
 
 		submit = false;
-		if (!port[1].request)
+		if (!port_count(&port[1]))
 			submit = i915_guc_dequeue(engine);
 	} while (submit);
 }
@@ -1051,8 +1068,7 @@ static int guc_ads_create(struct intel_guc *guc)
 		dev_priv->engine[RCS]->status_page.ggtt_offset;
 
 	for_each_engine(engine, dev_priv, id)
-		blob->ads.eng_state_size[engine->guc_id] =
-			intel_lr_context_size(engine);
+		blob->ads.eng_state_size[engine->guc_id] = engine->context_size;
 
 	base = guc_ggtt_offset(vma);
 	blob->ads.scheduler_policies = base + ptr_offset(blob, policies);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 190f6aa5d15e..4cd9ee1ba332 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -720,9 +720,7 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	i915_reg_t high_frame, low_frame;
 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
-	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
-								pipe);
-	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
+	const struct drm_display_mode *mode = &dev->vblank[pipe].hwmode;
 	unsigned long irqflags;
 
 	htotal = mode->crtc_htotal;
@@ -779,13 +777,17 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	const struct drm_display_mode *mode = &crtc->base.hwmode;
+	const struct drm_display_mode *mode;
+	struct drm_vblank_crtc *vblank;
 	enum pipe pipe = crtc->pipe;
 	int position, vtotal;
 
 	if (!crtc->active)
 		return -1;
 
+	vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
+	mode = &vblank->hwmode;
+
 	vtotal = mode->crtc_vtotal;
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 		vtotal /= 2;
@@ -827,10 +829,10 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 	return (position + crtc->scanline_offset) % vtotal;
 }
 
-static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
-				    unsigned int flags, int *vpos, int *hpos,
-				    ktime_t *stime, ktime_t *etime,
-				    const struct drm_display_mode *mode)
+static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
+				     bool in_vblank_irq, int *vpos, int *hpos,
+				     ktime_t *stime, ktime_t *etime,
+				     const struct drm_display_mode *mode)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
@@ -838,13 +840,12 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 	int position;
 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 	bool in_vbl = true;
-	int ret = 0;
 	unsigned long irqflags;
 
 	if (WARN_ON(!mode->crtc_clock)) {
 		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 				 "pipe %c\n", pipe_name(pipe));
-		return 0;
+		return false;
 	}
 
 	htotal = mode->crtc_htotal;
@@ -859,8 +860,6 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 		vtotal /= 2;
 	}
 
-	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
-
 	/*
 	 * Lock uncore.lock, as we will do multiple timing critical raw
 	 * register reads, potentially with preemption disabled, so the
@@ -944,11 +943,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 		*hpos = position - (*vpos * htotal);
 	}
 
-	/* In vblank? */
-	if (in_vbl)
-		ret |= DRM_SCANOUTPOS_IN_VBLANK;
-
-	return ret;
+	return true;
 }
 
 int intel_get_crtc_scanline(struct intel_crtc *crtc)
@@ -964,37 +959,6 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc)
 	return position;
 }
 
-static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
-			      int *max_error,
-			      struct timeval *vblank_time,
-			      unsigned flags)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc;
-
-	if (pipe >= INTEL_INFO(dev_priv)->num_pipes) {
-		DRM_ERROR("Invalid crtc %u\n", pipe);
-		return -EINVAL;
-	}
-
-	/* Get drm_crtc to timestamp: */
-	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-	if (crtc == NULL) {
-		DRM_ERROR("Invalid crtc %u\n", pipe);
-		return -EINVAL;
-	}
-
-	if (!crtc->base.hwmode.crtc_clock) {
-		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
-		return -EBUSY;
-	}
-
-	/* Helper routine in DRM core does all the work: */
-	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
-						     vblank_time, flags,
-						     &crtc->base.hwmode);
-}
-
 static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
 {
 	u32 busy_up, busy_down, max_avg, min_avg;
@@ -1236,7 +1200,7 @@ out:
 static void ivybridge_parity_work(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
-		container_of(work, struct drm_i915_private, l3_parity.error_work);
+		container_of(work, typeof(*dev_priv), l3_parity.error_work);
 	u32 error_status, row, bank, subbank;
 	char *parity_event[6];
 	uint32_t misccpctl;
@@ -1353,14 +1317,16 @@ static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
 }
 
-static __always_inline void
+static void
 gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
 {
 	bool tasklet = false;
 
 	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
-		set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-		tasklet = true;
+		if (port_count(&engine->execlist_port[0])) {
+			__set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
+			tasklet = true;
+		}
 	}
 
 	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
@@ -2582,7 +2548,8 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			I915_WRITE(SDEIIR, iir);
 			ret = IRQ_HANDLED;
 
-			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
+			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
+			    HAS_PCH_CNP(dev_priv))
 				spt_irq_handler(dev_priv, iir);
 			else
 				cpt_irq_handler(dev_priv, iir);
@@ -4230,11 +4197,15 @@ static void i965_irq_uninstall(struct drm_device * dev)
 void intel_irq_init(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
+	int i;
 
 	intel_hpd_init_work(dev_priv);
 
 	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
+
 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
+	for (i = 0; i < MAX_L3_SLICES; ++i)
+		dev_priv->l3_parity.remap_info[i] = NULL;
 
 	if (HAS_GUC_SCHED(dev_priv))
 		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;
@@ -4291,7 +4262,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 
 	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
 
-	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
+	dev->driver->get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos;
 	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
 
 	if (IS_CHERRYVIEW(dev_priv)) {
@@ -4319,7 +4290,8 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 		dev->driver->disable_vblank = gen8_disable_vblank;
 		if (IS_GEN9_LP(dev_priv))
 			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
-		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
+		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
+			 HAS_PCH_CNP(dev_priv))
 			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
 		else
 			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
@@ -4360,6 +4332,20 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 }
 
 /**
+ * intel_irq_fini - deinitializes IRQ support
+ * @i915: i915 device instance
+ *
+ * This function deinitializes all the IRQ support.
+ */
+void intel_irq_fini(struct drm_i915_private *i915)
+{
+	int i;
+
+	for (i = 0; i < MAX_L3_SLICES; ++i)
+		kfree(i915->l3_parity.remap_info[i]);
+}
+
+/**
  * intel_irq_install - enables the hardware interrupt
  * @dev_priv: i915 device instance
  *
diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.c b/drivers/gpu/drm/i915/i915_oa_bdw.c
new file mode 100644
index 000000000000..d4462c2aaaee
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_bdw.c
@@ -0,0 +1,5376 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_bdw.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_DATA_PORT_READS_COALESCING,
+	METRIC_SET_ID_DATA_PORT_WRITES_COALESCING,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_L3_4,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER_1,
+	METRIC_SET_ID_SAMPLER_2,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_VME_PIPE,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_bdw = 22;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic_0_slices_0x01[] = {
+	{ _MMIO(0x9888), 0x143f000f },
+	{ _MMIO(0x9888), 0x14110014 },
+	{ _MMIO(0x9888), 0x14310014 },
+	{ _MMIO(0x9888), 0x14bf000f },
+	{ _MMIO(0x9888), 0x118a0317 },
+	{ _MMIO(0x9888), 0x13837be0 },
+	{ _MMIO(0x9888), 0x3b800060 },
+	{ _MMIO(0x9888), 0x3d800005 },
+	{ _MMIO(0x9888), 0x005c4000 },
+	{ _MMIO(0x9888), 0x065c8000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x003d8000 },
+	{ _MMIO(0x9888), 0x183d0800 },
+	{ _MMIO(0x9888), 0x0a3f0023 },
+	{ _MMIO(0x9888), 0x103f0000 },
+	{ _MMIO(0x9888), 0x00584000 },
+	{ _MMIO(0x9888), 0x08584000 },
+	{ _MMIO(0x9888), 0x0a5a4000 },
+	{ _MMIO(0x9888), 0x005b4000 },
+	{ _MMIO(0x9888), 0x0e5b8000 },
+	{ _MMIO(0x9888), 0x185b2400 },
+	{ _MMIO(0x9888), 0x0a1d4000 },
+	{ _MMIO(0x9888), 0x0c1f0800 },
+	{ _MMIO(0x9888), 0x0e1faa00 },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18380001 },
+	{ _MMIO(0x9888), 0x00392000 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a391000 },
+	{ _MMIO(0x9888), 0x00104000 },
+	{ _MMIO(0x9888), 0x08104000 },
+	{ _MMIO(0x9888), 0x00110030 },
+	{ _MMIO(0x9888), 0x08110031 },
+	{ _MMIO(0x9888), 0x10110000 },
+	{ _MMIO(0x9888), 0x00134000 },
+	{ _MMIO(0x9888), 0x16130020 },
+	{ _MMIO(0x9888), 0x06308000 },
+	{ _MMIO(0x9888), 0x08308000 },
+	{ _MMIO(0x9888), 0x06311800 },
+	{ _MMIO(0x9888), 0x08311880 },
+	{ _MMIO(0x9888), 0x10310000 },
+	{ _MMIO(0x9888), 0x0e334000 },
+	{ _MMIO(0x9888), 0x16330080 },
+	{ _MMIO(0x9888), 0x0abf1180 },
+	{ _MMIO(0x9888), 0x10bf0000 },
+	{ _MMIO(0x9888), 0x0ada8000 },
+	{ _MMIO(0x9888), 0x0a9d8000 },
+	{ _MMIO(0x9888), 0x109f0002 },
+	{ _MMIO(0x9888), 0x0ab94000 },
+	{ _MMIO(0x9888), 0x0d888000 },
+	{ _MMIO(0x9888), 0x038a0380 },
+	{ _MMIO(0x9888), 0x058a000e },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8a00a0 },
+	{ _MMIO(0x9888), 0x078a0000 },
+	{ _MMIO(0x9888), 0x098a0000 },
+	{ _MMIO(0x9888), 0x238b2820 },
+	{ _MMIO(0x9888), 0x258b2550 },
+	{ _MMIO(0x9888), 0x198c1000 },
+	{ _MMIO(0x9888), 0x0b8d8000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa0 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x0d831021 },
+	{ _MMIO(0x9888), 0x0f83572f },
+	{ _MMIO(0x9888), 0x01835680 },
+	{ _MMIO(0x9888), 0x0383002c },
+	{ _MMIO(0x9888), 0x11830000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830001 },
+	{ _MMIO(0x9888), 0x05830000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x05844000 },
+	{ _MMIO(0x9888), 0x1b80c137 },
+	{ _MMIO(0x9888), 0x1d80c147 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x15804000 },
+	{ _MMIO(0x9888), 0x4d801110 },
+	{ _MMIO(0x9888), 0x4f800331 },
+	{ _MMIO(0x9888), 0x43800802 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45801465 },
+	{ _MMIO(0x9888), 0x53801111 },
+	{ _MMIO(0x9888), 0x478014a5 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800ca5 },
+	{ _MMIO(0x9888), 0x41800003 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic_1_slices_0x02[] = {
+	{ _MMIO(0x9888), 0x143f000f },
+	{ _MMIO(0x9888), 0x14bf000f },
+	{ _MMIO(0x9888), 0x14910014 },
+	{ _MMIO(0x9888), 0x14b10014 },
+	{ _MMIO(0x9888), 0x118a0317 },
+	{ _MMIO(0x9888), 0x13837be0 },
+	{ _MMIO(0x9888), 0x3b800060 },
+	{ _MMIO(0x9888), 0x3d800005 },
+	{ _MMIO(0x9888), 0x0a3f0023 },
+	{ _MMIO(0x9888), 0x103f0000 },
+	{ _MMIO(0x9888), 0x0a5a4000 },
+	{ _MMIO(0x9888), 0x0a1d4000 },
+	{ _MMIO(0x9888), 0x0e1f8000 },
+	{ _MMIO(0x9888), 0x0a391000 },
+	{ _MMIO(0x9888), 0x00dc4000 },
+	{ _MMIO(0x9888), 0x06dc8000 },
+	{ _MMIO(0x9888), 0x08dcc000 },
+	{ _MMIO(0x9888), 0x00bd8000 },
+	{ _MMIO(0x9888), 0x18bd0800 },
+	{ _MMIO(0x9888), 0x0abf1180 },
+	{ _MMIO(0x9888), 0x10bf0000 },
+	{ _MMIO(0x9888), 0x00d84000 },
+	{ _MMIO(0x9888), 0x08d84000 },
+	{ _MMIO(0x9888), 0x0ada8000 },
+	{ _MMIO(0x9888), 0x00db4000 },
+	{ _MMIO(0x9888), 0x0edb8000 },
+	{ _MMIO(0x9888), 0x18db2400 },
+	{ _MMIO(0x9888), 0x0a9d8000 },
+	{ _MMIO(0x9888), 0x0c9f0800 },
+	{ _MMIO(0x9888), 0x0e9f2a00 },
+	{ _MMIO(0x9888), 0x109f0002 },
+	{ _MMIO(0x9888), 0x00b84000 },
+	{ _MMIO(0x9888), 0x0eb84000 },
+	{ _MMIO(0x9888), 0x16b84000 },
+	{ _MMIO(0x9888), 0x18b80001 },
+	{ _MMIO(0x9888), 0x00b92000 },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab94000 },
+	{ _MMIO(0x9888), 0x00904000 },
+	{ _MMIO(0x9888), 0x08904000 },
+	{ _MMIO(0x9888), 0x00910030 },
+	{ _MMIO(0x9888), 0x08910031 },
+	{ _MMIO(0x9888), 0x10910000 },
+	{ _MMIO(0x9888), 0x00934000 },
+	{ _MMIO(0x9888), 0x16930020 },
+	{ _MMIO(0x9888), 0x06b08000 },
+	{ _MMIO(0x9888), 0x08b08000 },
+	{ _MMIO(0x9888), 0x06b11800 },
+	{ _MMIO(0x9888), 0x08b11880 },
+	{ _MMIO(0x9888), 0x10b10000 },
+	{ _MMIO(0x9888), 0x0eb34000 },
+	{ _MMIO(0x9888), 0x16b30080 },
+	{ _MMIO(0x9888), 0x01888000 },
+	{ _MMIO(0x9888), 0x0d88b800 },
+	{ _MMIO(0x9888), 0x038a0380 },
+	{ _MMIO(0x9888), 0x058a000e },
+	{ _MMIO(0x9888), 0x1b8a0080 },
+	{ _MMIO(0x9888), 0x078a0000 },
+	{ _MMIO(0x9888), 0x098a0000 },
+	{ _MMIO(0x9888), 0x238b2840 },
+	{ _MMIO(0x9888), 0x258b26a0 },
+	{ _MMIO(0x9888), 0x018c4000 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c1100 },
+	{ _MMIO(0x9888), 0x018d2000 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8d8000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa0 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x0d831021 },
+	{ _MMIO(0x9888), 0x0f83572f },
+	{ _MMIO(0x9888), 0x01835680 },
+	{ _MMIO(0x9888), 0x0383002c },
+	{ _MMIO(0x9888), 0x11830000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830001 },
+	{ _MMIO(0x9888), 0x05830000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x05844000 },
+	{ _MMIO(0x9888), 0x1b80c137 },
+	{ _MMIO(0x9888), 0x1d80c147 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x15804000 },
+	{ _MMIO(0x9888), 0x4d801550 },
+	{ _MMIO(0x9888), 0x4f800331 },
+	{ _MMIO(0x9888), 0x43800802 },
+	{ _MMIO(0x9888), 0x51800400 },
+	{ _MMIO(0x9888), 0x458004a1 },
+	{ _MMIO(0x9888), 0x53805555 },
+	{ _MMIO(0x9888), 0x47800421 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f801421 },
+	{ _MMIO(0x9888), 0x41800845 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
+
+	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) {
+		regs[n] = mux_config_render_basic_0_slices_0x01;
+		lens[n] = ARRAY_SIZE(mux_config_render_basic_0_slices_0x01);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x02) {
+		regs[n] = mux_config_render_basic_1_slices_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_render_basic_1_slices_0x02);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic_0_slices_0x01[] = {
+	{ _MMIO(0x9888), 0x105c00e0 },
+	{ _MMIO(0x9888), 0x105800e0 },
+	{ _MMIO(0x9888), 0x103800e0 },
+	{ _MMIO(0x9888), 0x3580001a },
+	{ _MMIO(0x9888), 0x3b800060 },
+	{ _MMIO(0x9888), 0x3d800005 },
+	{ _MMIO(0x9888), 0x065c2100 },
+	{ _MMIO(0x9888), 0x0a5c0041 },
+	{ _MMIO(0x9888), 0x0c5c6600 },
+	{ _MMIO(0x9888), 0x005c6580 },
+	{ _MMIO(0x9888), 0x085c8000 },
+	{ _MMIO(0x9888), 0x0e5c8000 },
+	{ _MMIO(0x9888), 0x00580042 },
+	{ _MMIO(0x9888), 0x08582080 },
+	{ _MMIO(0x9888), 0x0c58004c },
+	{ _MMIO(0x9888), 0x0e582580 },
+	{ _MMIO(0x9888), 0x005b4000 },
+	{ _MMIO(0x9888), 0x185b1000 },
+	{ _MMIO(0x9888), 0x1a5b0104 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1faa00 },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x08380042 },
+	{ _MMIO(0x9888), 0x0a382080 },
+	{ _MMIO(0x9888), 0x0e38404c },
+	{ _MMIO(0x9888), 0x0238404b },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x16380000 },
+	{ _MMIO(0x9888), 0x18381145 },
+	{ _MMIO(0x9888), 0x04380000 },
+	{ _MMIO(0x9888), 0x0039a000 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x02392000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x238b02a0 },
+	{ _MMIO(0x9888), 0x258b5550 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x1f850a80 },
+	{ _MMIO(0x9888), 0x2185aaa0 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x03844000 },
+	{ _MMIO(0x9888), 0x17808137 },
+	{ _MMIO(0x9888), 0x1980c147 },
+	{ _MMIO(0x9888), 0x1b80c0e5 },
+	{ _MMIO(0x9888), 0x1d80c0e3 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x13804000 },
+	{ _MMIO(0x9888), 0x15800000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d801000 },
+	{ _MMIO(0x9888), 0x4f800111 },
+	{ _MMIO(0x9888), 0x43800062 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800062 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800062 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f801062 },
+	{ _MMIO(0x9888), 0x41801084 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic_2_slices_0x02[] = {
+	{ _MMIO(0x9888), 0x10dc00e0 },
+	{ _MMIO(0x9888), 0x10d800e0 },
+	{ _MMIO(0x9888), 0x10b800e0 },
+	{ _MMIO(0x9888), 0x3580001a },
+	{ _MMIO(0x9888), 0x3b800060 },
+	{ _MMIO(0x9888), 0x3d800005 },
+	{ _MMIO(0x9888), 0x06dc2100 },
+	{ _MMIO(0x9888), 0x0adc0041 },
+	{ _MMIO(0x9888), 0x0cdc6600 },
+	{ _MMIO(0x9888), 0x00dc6580 },
+	{ _MMIO(0x9888), 0x08dc8000 },
+	{ _MMIO(0x9888), 0x0edc8000 },
+	{ _MMIO(0x9888), 0x00d80042 },
+	{ _MMIO(0x9888), 0x08d82080 },
+	{ _MMIO(0x9888), 0x0cd8004c },
+	{ _MMIO(0x9888), 0x0ed82580 },
+	{ _MMIO(0x9888), 0x00db4000 },
+	{ _MMIO(0x9888), 0x18db1000 },
+	{ _MMIO(0x9888), 0x1adb0104 },
+	{ _MMIO(0x9888), 0x0c9fa800 },
+	{ _MMIO(0x9888), 0x0e9faa00 },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x08b80042 },
+	{ _MMIO(0x9888), 0x0ab82080 },
+	{ _MMIO(0x9888), 0x0eb8404c },
+	{ _MMIO(0x9888), 0x02b8404b },
+	{ _MMIO(0x9888), 0x00b84000 },
+	{ _MMIO(0x9888), 0x16b80000 },
+	{ _MMIO(0x9888), 0x18b81145 },
+	{ _MMIO(0x9888), 0x04b80000 },
+	{ _MMIO(0x9888), 0x00b9a000 },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x02b92000 },
+	{ _MMIO(0x9888), 0x01888000 },
+	{ _MMIO(0x9888), 0x0d88f800 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x238b0540 },
+	{ _MMIO(0x9888), 0x258baaa0 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x018c4000 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c5500 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x018da000 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x038d2000 },
+	{ _MMIO(0x9888), 0x1f850a80 },
+	{ _MMIO(0x9888), 0x2185aaa0 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x03844000 },
+	{ _MMIO(0x9888), 0x17808137 },
+	{ _MMIO(0x9888), 0x1980c147 },
+	{ _MMIO(0x9888), 0x1b80c0e5 },
+	{ _MMIO(0x9888), 0x1d80c0e3 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x13804000 },
+	{ _MMIO(0x9888), 0x15800000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d805000 },
+	{ _MMIO(0x9888), 0x4f800555 },
+	{ _MMIO(0x9888), 0x43800062 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800062 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800062 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800062 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
+
+	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) {
+		regs[n] = mux_config_compute_basic_0_slices_0x01;
+		lens[n] = ARRAY_SIZE(mux_config_compute_basic_0_slices_0x01);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x02) {
+		regs[n] = mux_config_compute_basic_2_slices_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_compute_basic_2_slices_0x02);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0a1e0000 },
+	{ _MMIO(0x9888), 0x0c1f000f },
+	{ _MMIO(0x9888), 0x10176800 },
+	{ _MMIO(0x9888), 0x1191001f },
+	{ _MMIO(0x9888), 0x0b880320 },
+	{ _MMIO(0x9888), 0x01890c40 },
+	{ _MMIO(0x9888), 0x118a1c00 },
+	{ _MMIO(0x9888), 0x118d7c00 },
+	{ _MMIO(0x9888), 0x118e0020 },
+	{ _MMIO(0x9888), 0x118f4c00 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x13900001 },
+	{ _MMIO(0x9888), 0x065c4000 },
+	{ _MMIO(0x9888), 0x0c3d8000 },
+	{ _MMIO(0x9888), 0x06584000 },
+	{ _MMIO(0x9888), 0x0c5b4000 },
+	{ _MMIO(0x9888), 0x081e0040 },
+	{ _MMIO(0x9888), 0x0e1e0000 },
+	{ _MMIO(0x9888), 0x021f5400 },
+	{ _MMIO(0x9888), 0x001f0000 },
+	{ _MMIO(0x9888), 0x101f0010 },
+	{ _MMIO(0x9888), 0x0e1f0080 },
+	{ _MMIO(0x9888), 0x0c384000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x0c13c000 },
+	{ _MMIO(0x9888), 0x06164000 },
+	{ _MMIO(0x9888), 0x06170012 },
+	{ _MMIO(0x9888), 0x00170000 },
+	{ _MMIO(0x9888), 0x01910005 },
+	{ _MMIO(0x9888), 0x07880002 },
+	{ _MMIO(0x9888), 0x01880c00 },
+	{ _MMIO(0x9888), 0x0f880000 },
+	{ _MMIO(0x9888), 0x0d880000 },
+	{ _MMIO(0x9888), 0x05880000 },
+	{ _MMIO(0x9888), 0x09890032 },
+	{ _MMIO(0x9888), 0x078a0800 },
+	{ _MMIO(0x9888), 0x0f8a0a00 },
+	{ _MMIO(0x9888), 0x198a4000 },
+	{ _MMIO(0x9888), 0x1b8a2000 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x038a4000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x238b54c0 },
+	{ _MMIO(0x9888), 0x258baa55 },
+	{ _MMIO(0x9888), 0x278b0019 },
+	{ _MMIO(0x9888), 0x198c0100 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x0f8d0015 },
+	{ _MMIO(0x9888), 0x018d1000 },
+	{ _MMIO(0x9888), 0x098d8000 },
+	{ _MMIO(0x9888), 0x0b8df000 },
+	{ _MMIO(0x9888), 0x0d8d3000 },
+	{ _MMIO(0x9888), 0x038de000 },
+	{ _MMIO(0x9888), 0x058d3000 },
+	{ _MMIO(0x9888), 0x0d8e0004 },
+	{ _MMIO(0x9888), 0x058e000c },
+	{ _MMIO(0x9888), 0x098e0000 },
+	{ _MMIO(0x9888), 0x078e0000 },
+	{ _MMIO(0x9888), 0x038e0000 },
+	{ _MMIO(0x9888), 0x0b8f0020 },
+	{ _MMIO(0x9888), 0x198f0c00 },
+	{ _MMIO(0x9888), 0x078f8000 },
+	{ _MMIO(0x9888), 0x098f4000 },
+	{ _MMIO(0x9888), 0x0b900980 },
+	{ _MMIO(0x9888), 0x03900d80 },
+	{ _MMIO(0x9888), 0x01900000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d801111 },
+	{ _MMIO(0x9888), 0x3d800800 },
+	{ _MMIO(0x9888), 0x4f801011 },
+	{ _MMIO(0x9888), 0x43800443 },
+	{ _MMIO(0x9888), 0x51801111 },
+	{ _MMIO(0x9888), 0x45800422 },
+	{ _MMIO(0x9888), 0x53801111 },
+	{ _MMIO(0x9888), 0x47800c60 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800422 },
+	{ _MMIO(0x9888), 0x41800021 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x198b0343 },
+	{ _MMIO(0x9888), 0x13845800 },
+	{ _MMIO(0x9888), 0x15840018 },
+	{ _MMIO(0x9888), 0x3580001a },
+	{ _MMIO(0x9888), 0x038b6300 },
+	{ _MMIO(0x9888), 0x058b6b62 },
+	{ _MMIO(0x9888), 0x078b006a },
+	{ _MMIO(0x9888), 0x118b0000 },
+	{ _MMIO(0x9888), 0x238b0000 },
+	{ _MMIO(0x9888), 0x258b0000 },
+	{ _MMIO(0x9888), 0x1f85a080 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385000a },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x01840018 },
+	{ _MMIO(0x9888), 0x07844c80 },
+	{ _MMIO(0x9888), 0x09840d9a },
+	{ _MMIO(0x9888), 0x0b840e9c },
+	{ _MMIO(0x9888), 0x0d840f9e },
+	{ _MMIO(0x9888), 0x0f840010 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x03848000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x2f8000e5 },
+	{ _MMIO(0x9888), 0x138080e3 },
+	{ _MMIO(0x9888), 0x1580c0e1 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x11804000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f804000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800800 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800842 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800842 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47801042 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800084 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x198b0343 },
+	{ _MMIO(0x9888), 0x13845400 },
+	{ _MMIO(0x9888), 0x3580001a },
+	{ _MMIO(0x9888), 0x3d800805 },
+	{ _MMIO(0x9888), 0x038b6300 },
+	{ _MMIO(0x9888), 0x058b6b62 },
+	{ _MMIO(0x9888), 0x078b006a },
+	{ _MMIO(0x9888), 0x118b0000 },
+	{ _MMIO(0x9888), 0x238b0000 },
+	{ _MMIO(0x9888), 0x258b0000 },
+	{ _MMIO(0x9888), 0x1f85a080 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x23850002 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x01840010 },
+	{ _MMIO(0x9888), 0x07844880 },
+	{ _MMIO(0x9888), 0x09840992 },
+	{ _MMIO(0x9888), 0x0b840a94 },
+	{ _MMIO(0x9888), 0x0d840b96 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x03848000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x2d800147 },
+	{ _MMIO(0x9888), 0x2f8000e5 },
+	{ _MMIO(0x9888), 0x138080e3 },
+	{ _MMIO(0x9888), 0x1580c0e1 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x11804000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f800000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800842 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800842 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47801082 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800084 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_0_subslices_0x01[] = {
+	{ _MMIO(0x9888), 0x143d0160 },
+	{ _MMIO(0x9888), 0x163d2800 },
+	{ _MMIO(0x9888), 0x183d0120 },
+	{ _MMIO(0x9888), 0x105800e0 },
+	{ _MMIO(0x9888), 0x005cc000 },
+	{ _MMIO(0x9888), 0x065c8000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0c5cc000 },
+	{ _MMIO(0x9888), 0x0e5cc000 },
+	{ _MMIO(0x9888), 0x025cc000 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x003d0011 },
+	{ _MMIO(0x9888), 0x063d0900 },
+	{ _MMIO(0x9888), 0x083d0a13 },
+	{ _MMIO(0x9888), 0x0a3d0b15 },
+	{ _MMIO(0x9888), 0x0c3d2317 },
+	{ _MMIO(0x9888), 0x043d21b7 },
+	{ _MMIO(0x9888), 0x103d0000 },
+	{ _MMIO(0x9888), 0x0e3d0000 },
+	{ _MMIO(0x9888), 0x1a3d0000 },
+	{ _MMIO(0x9888), 0x0e5825c1 },
+	{ _MMIO(0x9888), 0x00586100 },
+	{ _MMIO(0x9888), 0x0258204c },
+	{ _MMIO(0x9888), 0x06588000 },
+	{ _MMIO(0x9888), 0x0858c000 },
+	{ _MMIO(0x9888), 0x0a58c000 },
+	{ _MMIO(0x9888), 0x0c58c000 },
+	{ _MMIO(0x9888), 0x0458c000 },
+	{ _MMIO(0x9888), 0x005b4000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x185b5400 },
+	{ _MMIO(0x9888), 0x1a5b0155 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1faa2a },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18381555 },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x04384000 },
+	{ _MMIO(0x9888), 0x06384000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0039a000 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x238b2aa0 },
+	{ _MMIO(0x9888), 0x258b5551 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa2 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_2_subslices_0x02[] = {
+	{ _MMIO(0x9888), 0x105c00e0 },
+	{ _MMIO(0x9888), 0x145b0160 },
+	{ _MMIO(0x9888), 0x165b2800 },
+	{ _MMIO(0x9888), 0x185b0120 },
+	{ _MMIO(0x9888), 0x0e5c25c1 },
+	{ _MMIO(0x9888), 0x005c6100 },
+	{ _MMIO(0x9888), 0x025c204c },
+	{ _MMIO(0x9888), 0x065c8000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0c5cc000 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x005b0011 },
+	{ _MMIO(0x9888), 0x065b0900 },
+	{ _MMIO(0x9888), 0x085b0a13 },
+	{ _MMIO(0x9888), 0x0a5b0b15 },
+	{ _MMIO(0x9888), 0x0c5b2317 },
+	{ _MMIO(0x9888), 0x045b21b7 },
+	{ _MMIO(0x9888), 0x105b0000 },
+	{ _MMIO(0x9888), 0x0e5b0000 },
+	{ _MMIO(0x9888), 0x1a5b0000 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1faa2a },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18381555 },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x04384000 },
+	{ _MMIO(0x9888), 0x06384000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0039a000 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x238b2aa0 },
+	{ _MMIO(0x9888), 0x258b5551 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa2 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_4_subslices_0x04[] = {
+	{ _MMIO(0x9888), 0x103800e0 },
+	{ _MMIO(0x9888), 0x143a0160 },
+	{ _MMIO(0x9888), 0x163a2800 },
+	{ _MMIO(0x9888), 0x183a0120 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1faa2a },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x0e38a5c1 },
+	{ _MMIO(0x9888), 0x0038a100 },
+	{ _MMIO(0x9888), 0x0238204c },
+	{ _MMIO(0x9888), 0x16388000 },
+	{ _MMIO(0x9888), 0x183802aa },
+	{ _MMIO(0x9888), 0x04380000 },
+	{ _MMIO(0x9888), 0x06380000 },
+	{ _MMIO(0x9888), 0x08388000 },
+	{ _MMIO(0x9888), 0x0a388000 },
+	{ _MMIO(0x9888), 0x0039a000 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x003a0011 },
+	{ _MMIO(0x9888), 0x063a0900 },
+	{ _MMIO(0x9888), 0x083a0a13 },
+	{ _MMIO(0x9888), 0x0a3a0b15 },
+	{ _MMIO(0x9888), 0x0c3a2317 },
+	{ _MMIO(0x9888), 0x043a21b7 },
+	{ _MMIO(0x9888), 0x103a0000 },
+	{ _MMIO(0x9888), 0x0e3a0000 },
+	{ _MMIO(0x9888), 0x1a3a0000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x238b2aa0 },
+	{ _MMIO(0x9888), 0x258b5551 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa2 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_1_subslices_0x08[] = {
+	{ _MMIO(0x9888), 0x14bd0160 },
+	{ _MMIO(0x9888), 0x16bd2800 },
+	{ _MMIO(0x9888), 0x18bd0120 },
+	{ _MMIO(0x9888), 0x10d800e0 },
+	{ _MMIO(0x9888), 0x00dcc000 },
+	{ _MMIO(0x9888), 0x06dc8000 },
+	{ _MMIO(0x9888), 0x08dcc000 },
+	{ _MMIO(0x9888), 0x0adcc000 },
+	{ _MMIO(0x9888), 0x0cdcc000 },
+	{ _MMIO(0x9888), 0x0edcc000 },
+	{ _MMIO(0x9888), 0x02dcc000 },
+	{ _MMIO(0x9888), 0x04dcc000 },
+	{ _MMIO(0x9888), 0x00bd0011 },
+	{ _MMIO(0x9888), 0x06bd0900 },
+	{ _MMIO(0x9888), 0x08bd0a13 },
+	{ _MMIO(0x9888), 0x0abd0b15 },
+	{ _MMIO(0x9888), 0x0cbd2317 },
+	{ _MMIO(0x9888), 0x04bd21b7 },
+	{ _MMIO(0x9888), 0x10bd0000 },
+	{ _MMIO(0x9888), 0x0ebd0000 },
+	{ _MMIO(0x9888), 0x1abd0000 },
+	{ _MMIO(0x9888), 0x0ed825c1 },
+	{ _MMIO(0x9888), 0x00d86100 },
+	{ _MMIO(0x9888), 0x02d8204c },
+	{ _MMIO(0x9888), 0x06d88000 },
+	{ _MMIO(0x9888), 0x08d8c000 },
+	{ _MMIO(0x9888), 0x0ad8c000 },
+	{ _MMIO(0x9888), 0x0cd8c000 },
+	{ _MMIO(0x9888), 0x04d8c000 },
+	{ _MMIO(0x9888), 0x00db4000 },
+	{ _MMIO(0x9888), 0x0edb4000 },
+	{ _MMIO(0x9888), 0x18db5400 },
+	{ _MMIO(0x9888), 0x1adb0155 },
+	{ _MMIO(0x9888), 0x02db4000 },
+	{ _MMIO(0x9888), 0x04db4000 },
+	{ _MMIO(0x9888), 0x06db4000 },
+	{ _MMIO(0x9888), 0x08db4000 },
+	{ _MMIO(0x9888), 0x0adb4000 },
+	{ _MMIO(0x9888), 0x0c9fa800 },
+	{ _MMIO(0x9888), 0x0e9faa2a },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x00b84000 },
+	{ _MMIO(0x9888), 0x0eb84000 },
+	{ _MMIO(0x9888), 0x16b84000 },
+	{ _MMIO(0x9888), 0x18b81555 },
+	{ _MMIO(0x9888), 0x02b84000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b84000 },
+	{ _MMIO(0x9888), 0x08b84000 },
+	{ _MMIO(0x9888), 0x0ab84000 },
+	{ _MMIO(0x9888), 0x00b9a000 },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x01888000 },
+	{ _MMIO(0x9888), 0x0d88f800 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x238b5540 },
+	{ _MMIO(0x9888), 0x258baaa2 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x018c4000 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c5500 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x018da000 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa2 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_3_subslices_0x10[] = {
+	{ _MMIO(0x9888), 0x10dc00e0 },
+	{ _MMIO(0x9888), 0x14db0160 },
+	{ _MMIO(0x9888), 0x16db2800 },
+	{ _MMIO(0x9888), 0x18db0120 },
+	{ _MMIO(0x9888), 0x0edc25c1 },
+	{ _MMIO(0x9888), 0x00dc6100 },
+	{ _MMIO(0x9888), 0x02dc204c },
+	{ _MMIO(0x9888), 0x06dc8000 },
+	{ _MMIO(0x9888), 0x08dcc000 },
+	{ _MMIO(0x9888), 0x0adcc000 },
+	{ _MMIO(0x9888), 0x0cdcc000 },
+	{ _MMIO(0x9888), 0x04dcc000 },
+	{ _MMIO(0x9888), 0x00db0011 },
+	{ _MMIO(0x9888), 0x06db0900 },
+	{ _MMIO(0x9888), 0x08db0a13 },
+	{ _MMIO(0x9888), 0x0adb0b15 },
+	{ _MMIO(0x9888), 0x0cdb2317 },
+	{ _MMIO(0x9888), 0x04db21b7 },
+	{ _MMIO(0x9888), 0x10db0000 },
+	{ _MMIO(0x9888), 0x0edb0000 },
+	{ _MMIO(0x9888), 0x1adb0000 },
+	{ _MMIO(0x9888), 0x0c9fa800 },
+	{ _MMIO(0x9888), 0x0e9faa2a },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x00b84000 },
+	{ _MMIO(0x9888), 0x0eb84000 },
+	{ _MMIO(0x9888), 0x16b84000 },
+	{ _MMIO(0x9888), 0x18b81555 },
+	{ _MMIO(0x9888), 0x02b84000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b84000 },
+	{ _MMIO(0x9888), 0x08b84000 },
+	{ _MMIO(0x9888), 0x0ab84000 },
+	{ _MMIO(0x9888), 0x00b9a000 },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x01888000 },
+	{ _MMIO(0x9888), 0x0d88f800 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x238b5540 },
+	{ _MMIO(0x9888), 0x258baaa2 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x018c4000 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c5500 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x018da000 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa2 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_5_subslices_0x20[] = {
+	{ _MMIO(0x9888), 0x10b800e0 },
+	{ _MMIO(0x9888), 0x14ba0160 },
+	{ _MMIO(0x9888), 0x16ba2800 },
+	{ _MMIO(0x9888), 0x18ba0120 },
+	{ _MMIO(0x9888), 0x0c9fa800 },
+	{ _MMIO(0x9888), 0x0e9faa2a },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x0eb8a5c1 },
+	{ _MMIO(0x9888), 0x00b8a100 },
+	{ _MMIO(0x9888), 0x02b8204c },
+	{ _MMIO(0x9888), 0x16b88000 },
+	{ _MMIO(0x9888), 0x18b802aa },
+	{ _MMIO(0x9888), 0x04b80000 },
+	{ _MMIO(0x9888), 0x06b80000 },
+	{ _MMIO(0x9888), 0x08b88000 },
+	{ _MMIO(0x9888), 0x0ab88000 },
+	{ _MMIO(0x9888), 0x00b9a000 },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x00ba0011 },
+	{ _MMIO(0x9888), 0x06ba0900 },
+	{ _MMIO(0x9888), 0x08ba0a13 },
+	{ _MMIO(0x9888), 0x0aba0b15 },
+	{ _MMIO(0x9888), 0x0cba2317 },
+	{ _MMIO(0x9888), 0x04ba21b7 },
+	{ _MMIO(0x9888), 0x10ba0000 },
+	{ _MMIO(0x9888), 0x0eba0000 },
+	{ _MMIO(0x9888), 0x1aba0000 },
+	{ _MMIO(0x9888), 0x01888000 },
+	{ _MMIO(0x9888), 0x0d88f800 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x238b5540 },
+	{ _MMIO(0x9888), 0x258baaa2 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x018c4000 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c5500 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x018da000 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa2 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800000 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 6);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 6);
+
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
+		regs[n] = mux_config_compute_extended_0_subslices_0x01;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_0_subslices_0x01);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x08) {
+		regs[n] = mux_config_compute_extended_1_subslices_0x08;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_1_subslices_0x08);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x02) {
+		regs[n] = mux_config_compute_extended_2_subslices_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_2_subslices_0x02);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x10) {
+		regs[n] = mux_config_compute_extended_3_subslices_0x10;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_3_subslices_0x10);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x04) {
+		regs[n] = mux_config_compute_extended_4_subslices_0x04;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_4_subslices_0x04);
+		n++;
+	}
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x20) {
+		regs[n] = mux_config_compute_extended_5_subslices_0x20;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_5_subslices_0x20);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x143f00b3 },
+	{ _MMIO(0x9888), 0x14bf00b3 },
+	{ _MMIO(0x9888), 0x138303c0 },
+	{ _MMIO(0x9888), 0x3b800060 },
+	{ _MMIO(0x9888), 0x3d800805 },
+	{ _MMIO(0x9888), 0x003f0029 },
+	{ _MMIO(0x9888), 0x063f1400 },
+	{ _MMIO(0x9888), 0x083f1225 },
+	{ _MMIO(0x9888), 0x0e3f1327 },
+	{ _MMIO(0x9888), 0x103f0000 },
+	{ _MMIO(0x9888), 0x005a4000 },
+	{ _MMIO(0x9888), 0x065a8000 },
+	{ _MMIO(0x9888), 0x085ac000 },
+	{ _MMIO(0x9888), 0x0e5ac000 },
+	{ _MMIO(0x9888), 0x001d4000 },
+	{ _MMIO(0x9888), 0x061d8000 },
+	{ _MMIO(0x9888), 0x081dc000 },
+	{ _MMIO(0x9888), 0x0e1dc000 },
+	{ _MMIO(0x9888), 0x0c1f0800 },
+	{ _MMIO(0x9888), 0x0e1f2a00 },
+	{ _MMIO(0x9888), 0x101f0280 },
+	{ _MMIO(0x9888), 0x00391000 },
+	{ _MMIO(0x9888), 0x06394000 },
+	{ _MMIO(0x9888), 0x08395000 },
+	{ _MMIO(0x9888), 0x0e395000 },
+	{ _MMIO(0x9888), 0x0abf1429 },
+	{ _MMIO(0x9888), 0x0cbf1225 },
+	{ _MMIO(0x9888), 0x00bf1380 },
+	{ _MMIO(0x9888), 0x02bf0026 },
+	{ _MMIO(0x9888), 0x10bf0000 },
+	{ _MMIO(0x9888), 0x0adac000 },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x00da8000 },
+	{ _MMIO(0x9888), 0x02da4000 },
+	{ _MMIO(0x9888), 0x0a9dc000 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x009d8000 },
+	{ _MMIO(0x9888), 0x029d4000 },
+	{ _MMIO(0x9888), 0x0e9f8000 },
+	{ _MMIO(0x9888), 0x109f002a },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0ab95000 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x00b94000 },
+	{ _MMIO(0x9888), 0x02b91000 },
+	{ _MMIO(0x9888), 0x0d88c000 },
+	{ _MMIO(0x9888), 0x0f880003 },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8a8020 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x238b0520 },
+	{ _MMIO(0x9888), 0x258ba950 },
+	{ _MMIO(0x9888), 0x278b0016 },
+	{ _MMIO(0x9888), 0x198c5400 },
+	{ _MMIO(0x9888), 0x1b8c0001 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038d2000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaa0 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x03835180 },
+	{ _MMIO(0x9888), 0x05834022 },
+	{ _MMIO(0x9888), 0x11830000 },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x07830000 },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x05844000 },
+	{ _MMIO(0x9888), 0x1b80c137 },
+	{ _MMIO(0x9888), 0x1d80c147 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x15804000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d801000 },
+	{ _MMIO(0x9888), 0x4f800111 },
+	{ _MMIO(0x9888), 0x43800842 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800840 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800800 },
+	{ _MMIO(0x9888), 0x418014a2 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_data_port_reads_coalescing[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0xba98ba98 },
+	{ _MMIO(0x2748), 0xba98ba98 },
+	{ _MMIO(0x2744), 0x00003377 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fff2 },
+	{ _MMIO(0x2774), 0x00007ff0 },
+	{ _MMIO(0x2778), 0x0007ffe2 },
+	{ _MMIO(0x277c), 0x00007ff0 },
+	{ _MMIO(0x2780), 0x0007ffc2 },
+	{ _MMIO(0x2784), 0x00007ff0 },
+	{ _MMIO(0x2788), 0x0007ff82 },
+	{ _MMIO(0x278c), 0x00007ff0 },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000bfef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000bfdf },
+	{ _MMIO(0x27a0), 0x0007fffa },
+	{ _MMIO(0x27a4), 0x0000bfbf },
+	{ _MMIO(0x27a8), 0x0007fffa },
+	{ _MMIO(0x27ac), 0x0000bf7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_data_port_reads_coalescing[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_data_port_reads_coalescing_0_subslices_0x01[] = {
+	{ _MMIO(0x9888), 0x103d0005 },
+	{ _MMIO(0x9888), 0x163d240b },
+	{ _MMIO(0x9888), 0x1058022f },
+	{ _MMIO(0x9888), 0x185b5520 },
+	{ _MMIO(0x9888), 0x198b0003 },
+	{ _MMIO(0x9888), 0x005cc000 },
+	{ _MMIO(0x9888), 0x065cc000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0c5cc000 },
+	{ _MMIO(0x9888), 0x0e5cc000 },
+	{ _MMIO(0x9888), 0x025c4000 },
+	{ _MMIO(0x9888), 0x045c8000 },
+	{ _MMIO(0x9888), 0x003d0000 },
+	{ _MMIO(0x9888), 0x063d00b0 },
+	{ _MMIO(0x9888), 0x083d0182 },
+	{ _MMIO(0x9888), 0x0a3d10a0 },
+	{ _MMIO(0x9888), 0x0c3d11a2 },
+	{ _MMIO(0x9888), 0x0e3d0000 },
+	{ _MMIO(0x9888), 0x183d0000 },
+	{ _MMIO(0x9888), 0x1a3d0000 },
+	{ _MMIO(0x9888), 0x0e582242 },
+	{ _MMIO(0x9888), 0x00586700 },
+	{ _MMIO(0x9888), 0x0258004f },
+	{ _MMIO(0x9888), 0x0658c000 },
+	{ _MMIO(0x9888), 0x0858c000 },
+	{ _MMIO(0x9888), 0x0a58c000 },
+	{ _MMIO(0x9888), 0x0c58c000 },
+	{ _MMIO(0x9888), 0x045b6300 },
+	{ _MMIO(0x9888), 0x105b0000 },
+	{ _MMIO(0x9888), 0x005b4000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x1a5b0155 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x0a5b0000 },
+	{ _MMIO(0x9888), 0x0c5b4000 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1faaa0 },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18381555 },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x04384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c384000 },
+	{ _MMIO(0x9888), 0x0039a000 },
+	{ _MMIO(0x9888), 0x0639a000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x02392000 },
+	{ _MMIO(0x9888), 0x04398000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x038b6300 },
+	{ _MMIO(0x9888), 0x058b0062 },
+	{ _MMIO(0x9888), 0x118b0000 },
+	{ _MMIO(0x9888), 0x238b02a0 },
+	{ _MMIO(0x9888), 0x258b5555 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d801000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800001 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800041 },
+};
+
+static int
+get_data_port_reads_coalescing_mux_config(struct drm_i915_private *dev_priv,
+					  const struct i915_oa_reg **regs,
+					  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
+		regs[n] = mux_config_data_port_reads_coalescing_0_subslices_0x01;
+		lens[n] = ARRAY_SIZE(mux_config_data_port_reads_coalescing_0_subslices_0x01);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_data_port_writes_coalescing[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0xba98ba98 },
+	{ _MMIO(0x2748), 0xba98ba98 },
+	{ _MMIO(0x2744), 0x00003377 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ff72 },
+	{ _MMIO(0x2774), 0x0000bfd0 },
+	{ _MMIO(0x2778), 0x0007ff62 },
+	{ _MMIO(0x277c), 0x0000bfd0 },
+	{ _MMIO(0x2780), 0x0007ff42 },
+	{ _MMIO(0x2784), 0x0000bfd0 },
+	{ _MMIO(0x2788), 0x0007ff02 },
+	{ _MMIO(0x278c), 0x0000bfd0 },
+	{ _MMIO(0x2790), 0x0005fff2 },
+	{ _MMIO(0x2794), 0x0000bfd0 },
+	{ _MMIO(0x2798), 0x0005ffe2 },
+	{ _MMIO(0x279c), 0x0000bfd0 },
+	{ _MMIO(0x27a0), 0x0005ffc2 },
+	{ _MMIO(0x27a4), 0x0000bfd0 },
+	{ _MMIO(0x27a8), 0x0005ff82 },
+	{ _MMIO(0x27ac), 0x0000bfd0 },
+};
+
+static const struct i915_oa_reg flex_eu_config_data_port_writes_coalescing[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_data_port_writes_coalescing_0_subslices_0x01[] = {
+	{ _MMIO(0x9888), 0x103d0005 },
+	{ _MMIO(0x9888), 0x143d0120 },
+	{ _MMIO(0x9888), 0x163d2400 },
+	{ _MMIO(0x9888), 0x1058022f },
+	{ _MMIO(0x9888), 0x105b0000 },
+	{ _MMIO(0x9888), 0x198b0003 },
+	{ _MMIO(0x9888), 0x005cc000 },
+	{ _MMIO(0x9888), 0x065cc000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0e5cc000 },
+	{ _MMIO(0x9888), 0x025c4000 },
+	{ _MMIO(0x9888), 0x045c8000 },
+	{ _MMIO(0x9888), 0x003d0000 },
+	{ _MMIO(0x9888), 0x063d0094 },
+	{ _MMIO(0x9888), 0x083d0182 },
+	{ _MMIO(0x9888), 0x0a3d1814 },
+	{ _MMIO(0x9888), 0x0e3d0000 },
+	{ _MMIO(0x9888), 0x183d0000 },
+	{ _MMIO(0x9888), 0x1a3d0000 },
+	{ _MMIO(0x9888), 0x0c3d0000 },
+	{ _MMIO(0x9888), 0x0e582242 },
+	{ _MMIO(0x9888), 0x00586700 },
+	{ _MMIO(0x9888), 0x0258004f },
+	{ _MMIO(0x9888), 0x0658c000 },
+	{ _MMIO(0x9888), 0x0858c000 },
+	{ _MMIO(0x9888), 0x0a58c000 },
+	{ _MMIO(0x9888), 0x045b6a80 },
+	{ _MMIO(0x9888), 0x005b4000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x185b5400 },
+	{ _MMIO(0x9888), 0x1a5b0141 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x0a5b0000 },
+	{ _MMIO(0x9888), 0x0c5b4000 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1faaa0 },
+	{ _MMIO(0x9888), 0x101f0282 },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18381415 },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x04384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c384000 },
+	{ _MMIO(0x9888), 0x0039a000 },
+	{ _MMIO(0x9888), 0x0639a000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x02392000 },
+	{ _MMIO(0x9888), 0x04398000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8a82a0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x038b6300 },
+	{ _MMIO(0x9888), 0x058b0062 },
+	{ _MMIO(0x9888), 0x118b0000 },
+	{ _MMIO(0x9888), 0x238b02a0 },
+	{ _MMIO(0x9888), 0x258b1555 },
+	{ _MMIO(0x9888), 0x278b0014 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x21852aaa },
+	{ _MMIO(0x9888), 0x23850028 },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830141 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0xd24), 0x00000000 },
+	{ _MMIO(0x9888), 0x4d801000 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f800001 },
+	{ _MMIO(0x9888), 0x43800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800420 },
+	{ _MMIO(0x9888), 0x3f800421 },
+	{ _MMIO(0x9888), 0x41800041 },
+};
+
+static int
+get_data_port_writes_coalescing_mux_config(struct drm_i915_private *dev_priv,
+					   const struct i915_oa_reg **regs,
+					   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
+		regs[n] = mux_config_data_port_writes_coalescing_0_subslices_0x01;
+		lens[n] = ARRAY_SIZE(mux_config_data_port_writes_coalescing_0_subslices_0x01);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fff7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x105c0232 },
+	{ _MMIO(0x9888), 0x10580232 },
+	{ _MMIO(0x9888), 0x10380232 },
+	{ _MMIO(0x9888), 0x10dc0232 },
+	{ _MMIO(0x9888), 0x10d80232 },
+	{ _MMIO(0x9888), 0x10b80232 },
+	{ _MMIO(0x9888), 0x118e4400 },
+	{ _MMIO(0x9888), 0x025c6080 },
+	{ _MMIO(0x9888), 0x045c004b },
+	{ _MMIO(0x9888), 0x005c8000 },
+	{ _MMIO(0x9888), 0x00582080 },
+	{ _MMIO(0x9888), 0x0258004b },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00aa },
+	{ _MMIO(0x9888), 0x04386080 },
+	{ _MMIO(0x9888), 0x0638404b },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a380000 },
+	{ _MMIO(0x9888), 0x0c380000 },
+	{ _MMIO(0x9888), 0x00398000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x0cdc25c1 },
+	{ _MMIO(0x9888), 0x0adcc000 },
+	{ _MMIO(0x9888), 0x0ad825c1 },
+	{ _MMIO(0x9888), 0x18db4000 },
+	{ _MMIO(0x9888), 0x1adb0001 },
+	{ _MMIO(0x9888), 0x0e9f8000 },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x0eb825c1 },
+	{ _MMIO(0x9888), 0x18b80154 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x0d88c000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x258baa05 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x198c5400 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x098dc000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x098e05c0 },
+	{ _MMIO(0x9888), 0x058e0000 },
+	{ _MMIO(0x9888), 0x198f0020 },
+	{ _MMIO(0x9888), 0x2185aa0a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x19835000 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x19808000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x51800040 },
+	{ _MMIO(0x9888), 0x43800400 },
+	{ _MMIO(0x9888), 0x45800800 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800c62 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f801042 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x418014a4 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x10bf03da },
+	{ _MMIO(0x9888), 0x14bf0001 },
+	{ _MMIO(0x9888), 0x12980340 },
+	{ _MMIO(0x9888), 0x12990340 },
+	{ _MMIO(0x9888), 0x0cbf1187 },
+	{ _MMIO(0x9888), 0x0ebf1205 },
+	{ _MMIO(0x9888), 0x00bf0500 },
+	{ _MMIO(0x9888), 0x02bf042b },
+	{ _MMIO(0x9888), 0x04bf002c },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x0edac000 },
+	{ _MMIO(0x9888), 0x00da8000 },
+	{ _MMIO(0x9888), 0x02dac000 },
+	{ _MMIO(0x9888), 0x04da4000 },
+	{ _MMIO(0x9888), 0x04983400 },
+	{ _MMIO(0x9888), 0x10980000 },
+	{ _MMIO(0x9888), 0x06990034 },
+	{ _MMIO(0x9888), 0x10990000 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x0e9dc000 },
+	{ _MMIO(0x9888), 0x009d8000 },
+	{ _MMIO(0x9888), 0x029dc000 },
+	{ _MMIO(0x9888), 0x049d4000 },
+	{ _MMIO(0x9888), 0x109f02a8 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f00ba },
+	{ _MMIO(0x9888), 0x0cb88000 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x0eb95000 },
+	{ _MMIO(0x9888), 0x00b94000 },
+	{ _MMIO(0x9888), 0x02b95000 },
+	{ _MMIO(0x9888), 0x04b91000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x0cba4000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x258b800a },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b5500 },
+	{ _MMIO(0x9888), 0x198c4000 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x47800000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800060 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x103f03da },
+	{ _MMIO(0x9888), 0x143f0001 },
+	{ _MMIO(0x9888), 0x12180340 },
+	{ _MMIO(0x9888), 0x12190340 },
+	{ _MMIO(0x9888), 0x0c3f1187 },
+	{ _MMIO(0x9888), 0x0e3f1205 },
+	{ _MMIO(0x9888), 0x003f0500 },
+	{ _MMIO(0x9888), 0x023f042b },
+	{ _MMIO(0x9888), 0x043f002c },
+	{ _MMIO(0x9888), 0x0c5ac000 },
+	{ _MMIO(0x9888), 0x0e5ac000 },
+	{ _MMIO(0x9888), 0x005a8000 },
+	{ _MMIO(0x9888), 0x025ac000 },
+	{ _MMIO(0x9888), 0x045a4000 },
+	{ _MMIO(0x9888), 0x04183400 },
+	{ _MMIO(0x9888), 0x10180000 },
+	{ _MMIO(0x9888), 0x06190034 },
+	{ _MMIO(0x9888), 0x10190000 },
+	{ _MMIO(0x9888), 0x0c1dc000 },
+	{ _MMIO(0x9888), 0x0e1dc000 },
+	{ _MMIO(0x9888), 0x001d8000 },
+	{ _MMIO(0x9888), 0x021dc000 },
+	{ _MMIO(0x9888), 0x041d4000 },
+	{ _MMIO(0x9888), 0x101f02a8 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00ba },
+	{ _MMIO(0x9888), 0x0c388000 },
+	{ _MMIO(0x9888), 0x0c395000 },
+	{ _MMIO(0x9888), 0x0e395000 },
+	{ _MMIO(0x9888), 0x00394000 },
+	{ _MMIO(0x9888), 0x02395000 },
+	{ _MMIO(0x9888), 0x04391000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x0c3a4000 },
+	{ _MMIO(0x9888), 0x1b8aa800 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x258b4005 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x47800000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800060 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x121b0340 },
+	{ _MMIO(0x9888), 0x103f0274 },
+	{ _MMIO(0x9888), 0x123f0000 },
+	{ _MMIO(0x9888), 0x129b0340 },
+	{ _MMIO(0x9888), 0x10bf0274 },
+	{ _MMIO(0x9888), 0x12bf0000 },
+	{ _MMIO(0x9888), 0x041b3400 },
+	{ _MMIO(0x9888), 0x101b0000 },
+	{ _MMIO(0x9888), 0x045c8000 },
+	{ _MMIO(0x9888), 0x0a3d4000 },
+	{ _MMIO(0x9888), 0x003f0080 },
+	{ _MMIO(0x9888), 0x023f0793 },
+	{ _MMIO(0x9888), 0x043f0014 },
+	{ _MMIO(0x9888), 0x04588000 },
+	{ _MMIO(0x9888), 0x005a8000 },
+	{ _MMIO(0x9888), 0x025ac000 },
+	{ _MMIO(0x9888), 0x045a4000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x001d8000 },
+	{ _MMIO(0x9888), 0x021dc000 },
+	{ _MMIO(0x9888), 0x041d4000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f002a },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x00394000 },
+	{ _MMIO(0x9888), 0x02395000 },
+	{ _MMIO(0x9888), 0x04399000 },
+	{ _MMIO(0x9888), 0x069b0034 },
+	{ _MMIO(0x9888), 0x109b0000 },
+	{ _MMIO(0x9888), 0x06dc4000 },
+	{ _MMIO(0x9888), 0x0cbd4000 },
+	{ _MMIO(0x9888), 0x0cbf0981 },
+	{ _MMIO(0x9888), 0x0ebf0a0f },
+	{ _MMIO(0x9888), 0x06d84000 },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x0edac000 },
+	{ _MMIO(0x9888), 0x0cdb4000 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x0e9dc000 },
+	{ _MMIO(0x9888), 0x109f02a8 },
+	{ _MMIO(0x9888), 0x0e9f0080 },
+	{ _MMIO(0x9888), 0x0cb84000 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x0eb95000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x258b8009 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x198c4000 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800c00 },
+	{ _MMIO(0x9888), 0x47800c63 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f8014a5 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800045 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_4[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_4[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_4[] = {
+	{ _MMIO(0x9888), 0x121a0340 },
+	{ _MMIO(0x9888), 0x103f0017 },
+	{ _MMIO(0x9888), 0x123f0020 },
+	{ _MMIO(0x9888), 0x129a0340 },
+	{ _MMIO(0x9888), 0x10bf0017 },
+	{ _MMIO(0x9888), 0x12bf0020 },
+	{ _MMIO(0x9888), 0x041a3400 },
+	{ _MMIO(0x9888), 0x101a0000 },
+	{ _MMIO(0x9888), 0x043b8000 },
+	{ _MMIO(0x9888), 0x0a3e0010 },
+	{ _MMIO(0x9888), 0x003f0200 },
+	{ _MMIO(0x9888), 0x023f0113 },
+	{ _MMIO(0x9888), 0x043f0014 },
+	{ _MMIO(0x9888), 0x02592000 },
+	{ _MMIO(0x9888), 0x005a8000 },
+	{ _MMIO(0x9888), 0x025ac000 },
+	{ _MMIO(0x9888), 0x045a4000 },
+	{ _MMIO(0x9888), 0x0a1c8000 },
+	{ _MMIO(0x9888), 0x001d8000 },
+	{ _MMIO(0x9888), 0x021dc000 },
+	{ _MMIO(0x9888), 0x041d4000 },
+	{ _MMIO(0x9888), 0x0a1e8000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f001a },
+	{ _MMIO(0x9888), 0x00394000 },
+	{ _MMIO(0x9888), 0x02395000 },
+	{ _MMIO(0x9888), 0x04391000 },
+	{ _MMIO(0x9888), 0x069a0034 },
+	{ _MMIO(0x9888), 0x109a0000 },
+	{ _MMIO(0x9888), 0x06bb4000 },
+	{ _MMIO(0x9888), 0x0abe0040 },
+	{ _MMIO(0x9888), 0x0cbf0984 },
+	{ _MMIO(0x9888), 0x0ebf0a02 },
+	{ _MMIO(0x9888), 0x02d94000 },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x0edac000 },
+	{ _MMIO(0x9888), 0x0c9c0400 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x0e9dc000 },
+	{ _MMIO(0x9888), 0x0c9e0400 },
+	{ _MMIO(0x9888), 0x109f02a8 },
+	{ _MMIO(0x9888), 0x0e9f0040 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x0eb95000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x258b8009 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x198c4000 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800800 },
+	{ _MMIO(0x9888), 0x47800842 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f801084 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800044 },
+};
+
+static int
+get_l3_4_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_4;
+	lens[n] = ARRAY_SIZE(mux_config_l3_4);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00006000 },
+	{ _MMIO(0x2774), 0x0000f3ff },
+	{ _MMIO(0x2778), 0x00001800 },
+	{ _MMIO(0x277c), 0x0000fcff },
+	{ _MMIO(0x2780), 0x00000600 },
+	{ _MMIO(0x2784), 0x0000ff3f },
+	{ _MMIO(0x2788), 0x00000180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000060 },
+	{ _MMIO(0x2794), 0x0000fff3 },
+	{ _MMIO(0x2798), 0x00000018 },
+	{ _MMIO(0x279c), 0x0000fffc },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x143b000e },
+	{ _MMIO(0x9888), 0x043c55c0 },
+	{ _MMIO(0x9888), 0x0a1e0280 },
+	{ _MMIO(0x9888), 0x0c1e0408 },
+	{ _MMIO(0x9888), 0x10390000 },
+	{ _MMIO(0x9888), 0x12397a1f },
+	{ _MMIO(0x9888), 0x14bb000e },
+	{ _MMIO(0x9888), 0x04bc5000 },
+	{ _MMIO(0x9888), 0x0a9e0296 },
+	{ _MMIO(0x9888), 0x0c9e0008 },
+	{ _MMIO(0x9888), 0x10b90000 },
+	{ _MMIO(0x9888), 0x12b97a1f },
+	{ _MMIO(0x9888), 0x063b0042 },
+	{ _MMIO(0x9888), 0x103b0000 },
+	{ _MMIO(0x9888), 0x083c0000 },
+	{ _MMIO(0x9888), 0x0a3e0040 },
+	{ _MMIO(0x9888), 0x043f8000 },
+	{ _MMIO(0x9888), 0x02594000 },
+	{ _MMIO(0x9888), 0x045a8000 },
+	{ _MMIO(0x9888), 0x0c1c0400 },
+	{ _MMIO(0x9888), 0x041d8000 },
+	{ _MMIO(0x9888), 0x081e02c0 },
+	{ _MMIO(0x9888), 0x0e1e0000 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1f0260 },
+	{ _MMIO(0x9888), 0x101f0014 },
+	{ _MMIO(0x9888), 0x003905e0 },
+	{ _MMIO(0x9888), 0x06390bc0 },
+	{ _MMIO(0x9888), 0x02390018 },
+	{ _MMIO(0x9888), 0x04394000 },
+	{ _MMIO(0x9888), 0x04bb0042 },
+	{ _MMIO(0x9888), 0x10bb0000 },
+	{ _MMIO(0x9888), 0x02bc05c0 },
+	{ _MMIO(0x9888), 0x08bc0000 },
+	{ _MMIO(0x9888), 0x0abe0004 },
+	{ _MMIO(0x9888), 0x02bf8000 },
+	{ _MMIO(0x9888), 0x02d91000 },
+	{ _MMIO(0x9888), 0x02da8000 },
+	{ _MMIO(0x9888), 0x089c8000 },
+	{ _MMIO(0x9888), 0x029d8000 },
+	{ _MMIO(0x9888), 0x089e8000 },
+	{ _MMIO(0x9888), 0x0e9e0000 },
+	{ _MMIO(0x9888), 0x0e9fa806 },
+	{ _MMIO(0x9888), 0x109f0142 },
+	{ _MMIO(0x9888), 0x08b90617 },
+	{ _MMIO(0x9888), 0x0ab90be0 },
+	{ _MMIO(0x9888), 0x02b94000 },
+	{ _MMIO(0x9888), 0x0d88f000 },
+	{ _MMIO(0x9888), 0x0f88000c },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x1b8a2800 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x238b52a0 },
+	{ _MMIO(0x9888), 0x258b6a95 },
+	{ _MMIO(0x9888), 0x278b0029 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c1500 },
+	{ _MMIO(0x9888), 0x1b8c0014 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x038d8000 },
+	{ _MMIO(0x9888), 0x058d2000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x4d800444 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f804000 },
+	{ _MMIO(0x9888), 0x43801080 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800084 },
+	{ _MMIO(0x9888), 0x53800044 },
+	{ _MMIO(0x9888), 0x47801080 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800000 },
+	{ _MMIO(0x9888), 0x41800840 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler_1[] = {
+	{ _MMIO(0x9888), 0x18921400 },
+	{ _MMIO(0x9888), 0x149500ab },
+	{ _MMIO(0x9888), 0x18b21400 },
+	{ _MMIO(0x9888), 0x14b500ab },
+	{ _MMIO(0x9888), 0x18d21400 },
+	{ _MMIO(0x9888), 0x14d500ab },
+	{ _MMIO(0x9888), 0x0cdc8000 },
+	{ _MMIO(0x9888), 0x0edc4000 },
+	{ _MMIO(0x9888), 0x02dcc000 },
+	{ _MMIO(0x9888), 0x04dcc000 },
+	{ _MMIO(0x9888), 0x1abd00a0 },
+	{ _MMIO(0x9888), 0x0abd8000 },
+	{ _MMIO(0x9888), 0x0cd88000 },
+	{ _MMIO(0x9888), 0x0ed84000 },
+	{ _MMIO(0x9888), 0x04d88000 },
+	{ _MMIO(0x9888), 0x1adb0050 },
+	{ _MMIO(0x9888), 0x04db8000 },
+	{ _MMIO(0x9888), 0x06db8000 },
+	{ _MMIO(0x9888), 0x08db8000 },
+	{ _MMIO(0x9888), 0x0adb4000 },
+	{ _MMIO(0x9888), 0x109f02a0 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f00aa },
+	{ _MMIO(0x9888), 0x18b82500 },
+	{ _MMIO(0x9888), 0x02b88000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b84000 },
+	{ _MMIO(0x9888), 0x08b84000 },
+	{ _MMIO(0x9888), 0x0ab84000 },
+	{ _MMIO(0x9888), 0x0cb88000 },
+	{ _MMIO(0x9888), 0x0cb98000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x00b98000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x1aba0200 },
+	{ _MMIO(0x9888), 0x02ba8000 },
+	{ _MMIO(0x9888), 0x0cba8000 },
+	{ _MMIO(0x9888), 0x04908000 },
+	{ _MMIO(0x9888), 0x04918000 },
+	{ _MMIO(0x9888), 0x04927300 },
+	{ _MMIO(0x9888), 0x10920000 },
+	{ _MMIO(0x9888), 0x1893000a },
+	{ _MMIO(0x9888), 0x0a934000 },
+	{ _MMIO(0x9888), 0x0a946000 },
+	{ _MMIO(0x9888), 0x0c959000 },
+	{ _MMIO(0x9888), 0x0e950098 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x04b04000 },
+	{ _MMIO(0x9888), 0x04b14000 },
+	{ _MMIO(0x9888), 0x04b20073 },
+	{ _MMIO(0x9888), 0x10b20000 },
+	{ _MMIO(0x9888), 0x04b38000 },
+	{ _MMIO(0x9888), 0x06b38000 },
+	{ _MMIO(0x9888), 0x08b34000 },
+	{ _MMIO(0x9888), 0x04b4c000 },
+	{ _MMIO(0x9888), 0x02b59890 },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x06d04000 },
+	{ _MMIO(0x9888), 0x06d14000 },
+	{ _MMIO(0x9888), 0x06d20073 },
+	{ _MMIO(0x9888), 0x10d20000 },
+	{ _MMIO(0x9888), 0x18d30020 },
+	{ _MMIO(0x9888), 0x02d38000 },
+	{ _MMIO(0x9888), 0x0cd34000 },
+	{ _MMIO(0x9888), 0x0ad48000 },
+	{ _MMIO(0x9888), 0x04d42000 },
+	{ _MMIO(0x9888), 0x0ed59000 },
+	{ _MMIO(0x9888), 0x00d59800 },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x0f88000e },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b5500 },
+	{ _MMIO(0x9888), 0x258b000a },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8d8000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x2185000a },
+	{ _MMIO(0x9888), 0x1b830150 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d848000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d808000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47801021 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800c64 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800c02 },
+};
+
+static int
+get_sampler_1_mux_config(struct drm_i915_private *dev_priv,
+			 const struct i915_oa_reg **regs,
+			 int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler_1;
+	lens[n] = ARRAY_SIZE(mux_config_sampler_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler_2[] = {
+	{ _MMIO(0x9888), 0x18121400 },
+	{ _MMIO(0x9888), 0x141500ab },
+	{ _MMIO(0x9888), 0x18321400 },
+	{ _MMIO(0x9888), 0x143500ab },
+	{ _MMIO(0x9888), 0x18521400 },
+	{ _MMIO(0x9888), 0x145500ab },
+	{ _MMIO(0x9888), 0x0c5c8000 },
+	{ _MMIO(0x9888), 0x0e5c4000 },
+	{ _MMIO(0x9888), 0x025cc000 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x1a3d00a0 },
+	{ _MMIO(0x9888), 0x0a3d8000 },
+	{ _MMIO(0x9888), 0x0c588000 },
+	{ _MMIO(0x9888), 0x0e584000 },
+	{ _MMIO(0x9888), 0x04588000 },
+	{ _MMIO(0x9888), 0x1a5b0050 },
+	{ _MMIO(0x9888), 0x045b8000 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b8000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x101f02a0 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00aa },
+	{ _MMIO(0x9888), 0x18382500 },
+	{ _MMIO(0x9888), 0x02388000 },
+	{ _MMIO(0x9888), 0x04384000 },
+	{ _MMIO(0x9888), 0x06384000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c388000 },
+	{ _MMIO(0x9888), 0x0c398000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x00398000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x1a3a0200 },
+	{ _MMIO(0x9888), 0x023a8000 },
+	{ _MMIO(0x9888), 0x0c3a8000 },
+	{ _MMIO(0x9888), 0x04108000 },
+	{ _MMIO(0x9888), 0x04118000 },
+	{ _MMIO(0x9888), 0x04127300 },
+	{ _MMIO(0x9888), 0x10120000 },
+	{ _MMIO(0x9888), 0x1813000a },
+	{ _MMIO(0x9888), 0x0a134000 },
+	{ _MMIO(0x9888), 0x0a146000 },
+	{ _MMIO(0x9888), 0x0c159000 },
+	{ _MMIO(0x9888), 0x0e150098 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04304000 },
+	{ _MMIO(0x9888), 0x04314000 },
+	{ _MMIO(0x9888), 0x04320073 },
+	{ _MMIO(0x9888), 0x10320000 },
+	{ _MMIO(0x9888), 0x04338000 },
+	{ _MMIO(0x9888), 0x06338000 },
+	{ _MMIO(0x9888), 0x08334000 },
+	{ _MMIO(0x9888), 0x0434c000 },
+	{ _MMIO(0x9888), 0x02359890 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x06504000 },
+	{ _MMIO(0x9888), 0x06514000 },
+	{ _MMIO(0x9888), 0x06520073 },
+	{ _MMIO(0x9888), 0x10520000 },
+	{ _MMIO(0x9888), 0x18530020 },
+	{ _MMIO(0x9888), 0x02538000 },
+	{ _MMIO(0x9888), 0x0c534000 },
+	{ _MMIO(0x9888), 0x0a548000 },
+	{ _MMIO(0x9888), 0x04542000 },
+	{ _MMIO(0x9888), 0x0e559000 },
+	{ _MMIO(0x9888), 0x00559800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x1b8aa000 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x258b0005 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x2185000a },
+	{ _MMIO(0x9888), 0x1b830150 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d848000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d808000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47801021 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800c64 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800c02 },
+};
+
+static int
+get_sampler_2_mux_config(struct drm_i915_private *dev_priv,
+			 const struct i915_oa_reg **regs,
+			 int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler_2;
+	lens[n] = ARRAY_SIZE(mux_config_sampler_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x0000fe7f },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000ffbf },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fff7 },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fff9 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x16154d60 },
+	{ _MMIO(0x9888), 0x16352e60 },
+	{ _MMIO(0x9888), 0x16554d60 },
+	{ _MMIO(0x9888), 0x16950000 },
+	{ _MMIO(0x9888), 0x16b50000 },
+	{ _MMIO(0x9888), 0x16d50000 },
+	{ _MMIO(0x9888), 0x005c8000 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x065c4000 },
+	{ _MMIO(0x9888), 0x083d8000 },
+	{ _MMIO(0x9888), 0x0a3d8000 },
+	{ _MMIO(0x9888), 0x0458c000 },
+	{ _MMIO(0x9888), 0x025b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00aa },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x04388000 },
+	{ _MMIO(0x9888), 0x06388000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c384000 },
+	{ _MMIO(0x9888), 0x00398000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x043a8000 },
+	{ _MMIO(0x9888), 0x063a8000 },
+	{ _MMIO(0x9888), 0x08138000 },
+	{ _MMIO(0x9888), 0x0a138000 },
+	{ _MMIO(0x9888), 0x06143000 },
+	{ _MMIO(0x9888), 0x0415cfc7 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x02338000 },
+	{ _MMIO(0x9888), 0x0c338000 },
+	{ _MMIO(0x9888), 0x04342000 },
+	{ _MMIO(0x9888), 0x06344000 },
+	{ _MMIO(0x9888), 0x0035c700 },
+	{ _MMIO(0x9888), 0x063500cf },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04538000 },
+	{ _MMIO(0x9888), 0x06538000 },
+	{ _MMIO(0x9888), 0x0454c000 },
+	{ _MMIO(0x9888), 0x0255cfc7 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06dc8000 },
+	{ _MMIO(0x9888), 0x08dc4000 },
+	{ _MMIO(0x9888), 0x0cdcc000 },
+	{ _MMIO(0x9888), 0x0edcc000 },
+	{ _MMIO(0x9888), 0x1abd00a8 },
+	{ _MMIO(0x9888), 0x0cd8c000 },
+	{ _MMIO(0x9888), 0x0ed84000 },
+	{ _MMIO(0x9888), 0x0edb8000 },
+	{ _MMIO(0x9888), 0x18db0800 },
+	{ _MMIO(0x9888), 0x1adb0254 },
+	{ _MMIO(0x9888), 0x0e9faa00 },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x0eb84000 },
+	{ _MMIO(0x9888), 0x16b84000 },
+	{ _MMIO(0x9888), 0x18b8156a },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x18baa000 },
+	{ _MMIO(0x9888), 0x1aba0002 },
+	{ _MMIO(0x9888), 0x16934000 },
+	{ _MMIO(0x9888), 0x1893000a },
+	{ _MMIO(0x9888), 0x0a947000 },
+	{ _MMIO(0x9888), 0x0c95c5c1 },
+	{ _MMIO(0x9888), 0x0e9500c3 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x0eb38000 },
+	{ _MMIO(0x9888), 0x16b30040 },
+	{ _MMIO(0x9888), 0x18b30020 },
+	{ _MMIO(0x9888), 0x06b48000 },
+	{ _MMIO(0x9888), 0x08b41000 },
+	{ _MMIO(0x9888), 0x0ab48000 },
+	{ _MMIO(0x9888), 0x06b5c500 },
+	{ _MMIO(0x9888), 0x08b500c3 },
+	{ _MMIO(0x9888), 0x0eb5c100 },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x16d31500 },
+	{ _MMIO(0x9888), 0x08d4e000 },
+	{ _MMIO(0x9888), 0x08d5c100 },
+	{ _MMIO(0x9888), 0x0ad5c3c5 },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x0d88f800 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x258baaa5 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c5500 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800c42 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800063 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800800 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f8014a4 },
+	{ _MMIO(0x9888), 0x41801042 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x0000fe7f },
+	{ _MMIO(0x2780), 0x00000000 },
+	{ _MMIO(0x2784), 0x0000ff9f },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000ffe7 },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fffb },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000fffd },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x16150000 },
+	{ _MMIO(0x9888), 0x16350000 },
+	{ _MMIO(0x9888), 0x16550000 },
+	{ _MMIO(0x9888), 0x16952e60 },
+	{ _MMIO(0x9888), 0x16b54d60 },
+	{ _MMIO(0x9888), 0x16d52e60 },
+	{ _MMIO(0x9888), 0x065c8000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0c5c4000 },
+	{ _MMIO(0x9888), 0x0e3d8000 },
+	{ _MMIO(0x9888), 0x183da000 },
+	{ _MMIO(0x9888), 0x06588000 },
+	{ _MMIO(0x9888), 0x08588000 },
+	{ _MMIO(0x9888), 0x0a584000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x185b5800 },
+	{ _MMIO(0x9888), 0x1a5b000a },
+	{ _MMIO(0x9888), 0x0e1faa00 },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18382a55 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x1a3a02a0 },
+	{ _MMIO(0x9888), 0x0e138000 },
+	{ _MMIO(0x9888), 0x16130500 },
+	{ _MMIO(0x9888), 0x06148000 },
+	{ _MMIO(0x9888), 0x08146000 },
+	{ _MMIO(0x9888), 0x0615c100 },
+	{ _MMIO(0x9888), 0x0815c500 },
+	{ _MMIO(0x9888), 0x0a1500c3 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x16335040 },
+	{ _MMIO(0x9888), 0x08349000 },
+	{ _MMIO(0x9888), 0x0a341000 },
+	{ _MMIO(0x9888), 0x083500c1 },
+	{ _MMIO(0x9888), 0x0a35c500 },
+	{ _MMIO(0x9888), 0x0c3500c3 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x1853002a },
+	{ _MMIO(0x9888), 0x0a54e000 },
+	{ _MMIO(0x9888), 0x0c55c500 },
+	{ _MMIO(0x9888), 0x0e55c1c3 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x00dc8000 },
+	{ _MMIO(0x9888), 0x02dcc000 },
+	{ _MMIO(0x9888), 0x04dc4000 },
+	{ _MMIO(0x9888), 0x04bd8000 },
+	{ _MMIO(0x9888), 0x06bd8000 },
+	{ _MMIO(0x9888), 0x02d8c000 },
+	{ _MMIO(0x9888), 0x02db8000 },
+	{ _MMIO(0x9888), 0x04db4000 },
+	{ _MMIO(0x9888), 0x06db4000 },
+	{ _MMIO(0x9888), 0x08db8000 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f00aa },
+	{ _MMIO(0x9888), 0x02b84000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b84000 },
+	{ _MMIO(0x9888), 0x08b84000 },
+	{ _MMIO(0x9888), 0x0ab88000 },
+	{ _MMIO(0x9888), 0x0cb88000 },
+	{ _MMIO(0x9888), 0x00b98000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x0aba8000 },
+	{ _MMIO(0x9888), 0x0cba8000 },
+	{ _MMIO(0x9888), 0x04938000 },
+	{ _MMIO(0x9888), 0x06938000 },
+	{ _MMIO(0x9888), 0x0494c000 },
+	{ _MMIO(0x9888), 0x0295cfc7 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x02b38000 },
+	{ _MMIO(0x9888), 0x08b38000 },
+	{ _MMIO(0x9888), 0x04b42000 },
+	{ _MMIO(0x9888), 0x06b41000 },
+	{ _MMIO(0x9888), 0x00b5c700 },
+	{ _MMIO(0x9888), 0x04b500cf },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x0ad38000 },
+	{ _MMIO(0x9888), 0x0cd38000 },
+	{ _MMIO(0x9888), 0x06d46000 },
+	{ _MMIO(0x9888), 0x04d5c700 },
+	{ _MMIO(0x9888), 0x06d500cf },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x258b555a },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x238b5500 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800882 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45801082 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x478014a5 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800002 },
+	{ _MMIO(0x9888), 0x41800c62 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+	{ _MMIO(0xe458), 0x00001000 },
+	{ _MMIO(0xe558), 0x00003002 },
+	{ _MMIO(0xe658), 0x00005004 },
+	{ _MMIO(0xe758), 0x00011010 },
+	{ _MMIO(0xe45c), 0x00050012 },
+	{ _MMIO(0xe55c), 0x00052051 },
+	{ _MMIO(0xe65c), 0x00000008 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x161503e0 },
+	{ _MMIO(0x9888), 0x163503e0 },
+	{ _MMIO(0x9888), 0x165503e0 },
+	{ _MMIO(0x9888), 0x169503e0 },
+	{ _MMIO(0x9888), 0x16b503e0 },
+	{ _MMIO(0x9888), 0x16d503e0 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x083d8000 },
+	{ _MMIO(0x9888), 0x04584000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5b8000 },
+	{ _MMIO(0x9888), 0x0e1f00a8 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c388000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x0c3a8000 },
+	{ _MMIO(0x9888), 0x08138000 },
+	{ _MMIO(0x9888), 0x06141000 },
+	{ _MMIO(0x9888), 0x041500c3 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x0a338000 },
+	{ _MMIO(0x9888), 0x06342000 },
+	{ _MMIO(0x9888), 0x0435c300 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x0c538000 },
+	{ _MMIO(0x9888), 0x06544000 },
+	{ _MMIO(0x9888), 0x065500c3 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x00dc8000 },
+	{ _MMIO(0x9888), 0x02dc4000 },
+	{ _MMIO(0x9888), 0x02bd8000 },
+	{ _MMIO(0x9888), 0x00d88000 },
+	{ _MMIO(0x9888), 0x02db4000 },
+	{ _MMIO(0x9888), 0x04db8000 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f0002 },
+	{ _MMIO(0x9888), 0x02b84000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b88000 },
+	{ _MMIO(0x9888), 0x00b98000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x06ba8000 },
+	{ _MMIO(0x9888), 0x02938000 },
+	{ _MMIO(0x9888), 0x04942000 },
+	{ _MMIO(0x9888), 0x0095c300 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x04b38000 },
+	{ _MMIO(0x9888), 0x04b44000 },
+	{ _MMIO(0x9888), 0x02b500c3 },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x06d38000 },
+	{ _MMIO(0x9888), 0x04d48000 },
+	{ _MMIO(0x9888), 0x02d5c300 },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x238b3500 },
+	{ _MMIO(0x9888), 0x258b0005 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x2185000a },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800c40 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41801482 },
+	{ _MMIO(0x9888), 0x31800000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00100030 },
+	{ _MMIO(0x2774), 0x0000fff9 },
+	{ _MMIO(0x2778), 0x00000002 },
+	{ _MMIO(0x277c), 0x0000fffc },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000fff3 },
+	{ _MMIO(0x2788), 0x00100180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000ff3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00008003 },
+};
+
+static const struct i915_oa_reg mux_config_vme_pipe[] = {
+	{ _MMIO(0x9888), 0x14100812 },
+	{ _MMIO(0x9888), 0x14125800 },
+	{ _MMIO(0x9888), 0x161200c0 },
+	{ _MMIO(0x9888), 0x14300812 },
+	{ _MMIO(0x9888), 0x14325800 },
+	{ _MMIO(0x9888), 0x163200c0 },
+	{ _MMIO(0x9888), 0x005c4000 },
+	{ _MMIO(0x9888), 0x065c8000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0c5cc000 },
+	{ _MMIO(0x9888), 0x003d8000 },
+	{ _MMIO(0x9888), 0x0e3d8000 },
+	{ _MMIO(0x9888), 0x183d2800 },
+	{ _MMIO(0x9888), 0x00584000 },
+	{ _MMIO(0x9888), 0x06588000 },
+	{ _MMIO(0x9888), 0x0858c000 },
+	{ _MMIO(0x9888), 0x005b4000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x185b9400 },
+	{ _MMIO(0x9888), 0x1a5b002a },
+	{ _MMIO(0x9888), 0x0c1f0800 },
+	{ _MMIO(0x9888), 0x0e1faa00 },
+	{ _MMIO(0x9888), 0x101f002a },
+	{ _MMIO(0x9888), 0x00384000 },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18380155 },
+	{ _MMIO(0x9888), 0x00392000 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x00100047 },
+	{ _MMIO(0x9888), 0x06101a80 },
+	{ _MMIO(0x9888), 0x10100000 },
+	{ _MMIO(0x9888), 0x0810c000 },
+	{ _MMIO(0x9888), 0x0811c000 },
+	{ _MMIO(0x9888), 0x08126151 },
+	{ _MMIO(0x9888), 0x10120000 },
+	{ _MMIO(0x9888), 0x00134000 },
+	{ _MMIO(0x9888), 0x0e134000 },
+	{ _MMIO(0x9888), 0x161300a0 },
+	{ _MMIO(0x9888), 0x0a301ac7 },
+	{ _MMIO(0x9888), 0x10300000 },
+	{ _MMIO(0x9888), 0x0c30c000 },
+	{ _MMIO(0x9888), 0x0c31c000 },
+	{ _MMIO(0x9888), 0x0c326151 },
+	{ _MMIO(0x9888), 0x10320000 },
+	{ _MMIO(0x9888), 0x16332a00 },
+	{ _MMIO(0x9888), 0x18330001 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8a2aa0 },
+	{ _MMIO(0x9888), 0x238b0020 },
+	{ _MMIO(0x9888), 0x258b5550 },
+	{ _MMIO(0x9888), 0x278b0001 },
+	{ _MMIO(0x9888), 0x1f850080 },
+	{ _MMIO(0x9888), 0x2185aaa0 },
+	{ _MMIO(0x9888), 0x23850002 },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830015 },
+	{ _MMIO(0x9888), 0x01844000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x11804000 },
+	{ _MMIO(0x9888), 0x17808000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3d800800 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800002 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800884 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800002 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+};
+
+static int
+get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
+			const struct i915_oa_reg **regs,
+			int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_vme_pipe;
+	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x198b0000 },
+	{ _MMIO(0x9888), 0x078b0066 },
+	{ _MMIO(0x9888), 0x118b0000 },
+	{ _MMIO(0x9888), 0x258b0000 },
+	{ _MMIO(0x9888), 0x21850008 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_bdw(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_DATA_PORT_READS_COALESCING:
+		dev_priv->perf.oa.n_mux_configs =
+			get_data_port_reads_coalescing_mux_config(dev_priv,
+								  dev_priv->perf.oa.mux_regs,
+								  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"DATA_PORT_READS_COALESCING\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_data_port_reads_coalescing;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_data_port_reads_coalescing);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_data_port_reads_coalescing;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_data_port_reads_coalescing);
+
+		return 0;
+	case METRIC_SET_ID_DATA_PORT_WRITES_COALESCING:
+		dev_priv->perf.oa.n_mux_configs =
+			get_data_port_writes_coalescing_mux_config(dev_priv,
+								   dev_priv->perf.oa.mux_regs,
+								   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"DATA_PORT_WRITES_COALESCING\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_data_port_writes_coalescing;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_data_port_writes_coalescing);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_data_port_writes_coalescing;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_data_port_writes_coalescing);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_L3_4:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_4_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_4\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_4;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_4);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_4;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_4);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_1_mux_config(dev_priv,
+						 dev_priv->perf.oa.mux_regs,
+						 dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler_1);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_2_mux_config(dev_priv,
+						 dev_priv->perf.oa.mux_regs,
+						 dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler_2);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_VME_PIPE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_vme_pipe_mux_config(dev_priv,
+						dev_priv->perf.oa.mux_regs,
+						dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_vme_pipe;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_vme_pipe);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_vme_pipe;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_vme_pipe);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "b541bd57-0e0f-4154-b4c0-5858010a2bf7",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "35fbc9b2-a891-40a6-a38d-022bb7057552",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "233d0544-fff7-4281-8291-e02f222aff72",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "2b255d48-2117-4fef-a8f7-f151e1d25a2c",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "f7fd3220-b466-4a4d-9f98-b0caf3f2394c",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "e99ccaca-821c-4df9-97a7-96bdb7204e43",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "27a364dc-8225-4ecb-b607-d6f1925598d9",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_data_port_reads_coalescing_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_DATA_PORT_READS_COALESCING);
+}
+
+static struct device_attribute dev_attr_data_port_reads_coalescing_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_data_port_reads_coalescing_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_data_port_reads_coalescing[] = {
+	&dev_attr_data_port_reads_coalescing_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_data_port_reads_coalescing = {
+	.name = "857fc630-2f09-4804-85f1-084adfadd5ab",
+	.attrs =  attrs_data_port_reads_coalescing,
+};
+
+static ssize_t
+show_data_port_writes_coalescing_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_DATA_PORT_WRITES_COALESCING);
+}
+
+static struct device_attribute dev_attr_data_port_writes_coalescing_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_data_port_writes_coalescing_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_data_port_writes_coalescing[] = {
+	&dev_attr_data_port_writes_coalescing_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_data_port_writes_coalescing = {
+	.name = "343ebc99-4a55-414c-8c17-d8e259cf5e20",
+	.attrs =  attrs_data_port_writes_coalescing,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "7bdafd88-a4fa-4ed5-bc09-1a977aa5be3e",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "9385ebb2-f34f-4aa5-aec5-7e9cbbea0f0b",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "446ae59b-ff2e-41c9-b49e-0184a54bf00a",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "84a7956f-1ea4-4d0d-837f-e39a0376e38c",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_l3_4_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_4);
+}
+
+static struct device_attribute dev_attr_l3_4_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_4_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_4[] = {
+	&dev_attr_l3_4_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_4 = {
+	.name = "92b493d9-df18-4bed-be06-5cac6f2a6f5f",
+	.attrs =  attrs_l3_4,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "14345c35-cc46-40d0-bb04-6ed1fbb43679",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_1);
+}
+
+static struct device_attribute dev_attr_sampler_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler_1[] = {
+	&dev_attr_sampler_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler_1 = {
+	.name = "f0c6ba37-d3d3-4211-91b5-226730312a54",
+	.attrs =  attrs_sampler_1,
+};
+
+static ssize_t
+show_sampler_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_2);
+}
+
+static struct device_attribute dev_attr_sampler_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler_2[] = {
+	&dev_attr_sampler_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler_2 = {
+	.name = "30bf3702-48cf-4bca-b412-7cf50bb2f564",
+	.attrs =  attrs_sampler_2,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "238bec85-df05-44f3-b905-d166712f2451",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "24bf02cd-8693-4583-981c-c4165b33da01",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "8fb61ba2-2fbb-454c-a136-2dec5a8a595e",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
+}
+
+static struct device_attribute dev_attr_vme_pipe_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_vme_pipe_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_vme_pipe[] = {
+	&dev_attr_vme_pipe_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_vme_pipe = {
+	.name = "e1743ca0-7fc8-410b-a066-de7bbb9280b7",
+	.attrs =  attrs_vme_pipe,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "d6de6f55-e526-4f79-a6a6-d7315c09044e",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_bdw(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_data_port_reads_coalescing_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_data_port_reads_coalescing);
+		if (ret)
+			goto error_data_port_reads_coalescing;
+	}
+	if (get_data_port_writes_coalescing_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_data_port_writes_coalescing);
+		if (ret)
+			goto error_data_port_writes_coalescing;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_4);
+		if (ret)
+			goto error_l3_4;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
+		if (ret)
+			goto error_sampler_1;
+	}
+	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
+		if (ret)
+			goto error_sampler_2;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+		if (ret)
+			goto error_vme_pipe;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+error_vme_pipe:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
+error_sampler_2:
+	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
+error_sampler_1:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
+error_l3_4:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_data_port_writes_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_writes_coalescing);
+error_data_port_writes_coalescing:
+	if (get_data_port_reads_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_reads_coalescing);
+error_data_port_reads_coalescing:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_bdw(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_data_port_reads_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_reads_coalescing);
+	if (get_data_port_writes_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_writes_coalescing);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
+	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.h b/drivers/gpu/drm/i915/i915_oa_bdw.h
new file mode 100644
index 000000000000..6363ff9f64c0
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_bdw.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_BDW_H__
+#define __I915_OA_BDW_H__
+
+extern int i915_oa_n_builtin_metric_sets_bdw;
+
+extern int i915_oa_select_metric_set_bdw(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_bdw(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_bdw(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.c b/drivers/gpu/drm/i915/i915_oa_bxt.c
new file mode 100644
index 000000000000..93864d8f32dd
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_bxt.c
@@ -0,0 +1,2690 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_bxt.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_bxt = 15;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic_0_sku_gte_0x03[] = {
+	{ _MMIO(0x9888), 0x166c00f0 },
+	{ _MMIO(0x9888), 0x12120280 },
+	{ _MMIO(0x9888), 0x12320280 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900c00 },
+	{ _MMIO(0x9888), 0x419000a0 },
+	{ _MMIO(0x9888), 0x002d1000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d5000 },
+	{ _MMIO(0x9888), 0x0a2d1000 },
+	{ _MMIO(0x9888), 0x0c2e0800 },
+	{ _MMIO(0x9888), 0x0e2e5900 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4c8000 },
+	{ _MMIO(0x9888), 0x0e4c4000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e2000 },
+	{ _MMIO(0x9888), 0x1c4f0010 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1a0fcc00 },
+	{ _MMIO(0x9888), 0x1c0f0002 },
+	{ _MMIO(0x9888), 0x1c2c0040 },
+	{ _MMIO(0x9888), 0x00101000 },
+	{ _MMIO(0x9888), 0x04101000 },
+	{ _MMIO(0x9888), 0x00114000 },
+	{ _MMIO(0x9888), 0x08114000 },
+	{ _MMIO(0x9888), 0x00120020 },
+	{ _MMIO(0x9888), 0x08120021 },
+	{ _MMIO(0x9888), 0x00141000 },
+	{ _MMIO(0x9888), 0x08141000 },
+	{ _MMIO(0x9888), 0x02308000 },
+	{ _MMIO(0x9888), 0x04302000 },
+	{ _MMIO(0x9888), 0x06318000 },
+	{ _MMIO(0x9888), 0x08318000 },
+	{ _MMIO(0x9888), 0x06320800 },
+	{ _MMIO(0x9888), 0x08320840 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x06344000 },
+	{ _MMIO(0x9888), 0x08344000 },
+	{ _MMIO(0x9888), 0x0d931831 },
+	{ _MMIO(0x9888), 0x0f939f3f },
+	{ _MMIO(0x9888), 0x01939e80 },
+	{ _MMIO(0x9888), 0x039303bc },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1993002a },
+	{ _MMIO(0x9888), 0x07930000 },
+	{ _MMIO(0x9888), 0x09930000 },
+	{ _MMIO(0x9888), 0x1d900177 },
+	{ _MMIO(0x9888), 0x1f900187 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x53901110 },
+	{ _MMIO(0x9888), 0x43900423 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x47900c02 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900020 },
+	{ _MMIO(0x9888), 0x59901111 },
+	{ _MMIO(0x9888), 0x4b900421 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x45900821 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	if (dev_priv->drm.pdev->revision >= 0x03) {
+		regs[n] = mux_config_render_basic_0_sku_gte_0x03;
+		lens[n] = ARRAY_SIZE(mux_config_render_basic_0_sku_gte_0x03);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x39900340 },
+	{ _MMIO(0x9888), 0x3f900c00 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x002d5000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d4000 },
+	{ _MMIO(0x9888), 0x0a2d1000 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x0c2e1400 },
+	{ _MMIO(0x9888), 0x0e2e5100 },
+	{ _MMIO(0x9888), 0x102e0114 },
+	{ _MMIO(0x9888), 0x044cc000 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4c8000 },
+	{ _MMIO(0x9888), 0x0e4c4000 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x004ea000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e2000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x004f6b42 },
+	{ _MMIO(0x9888), 0x064f6200 },
+	{ _MMIO(0x9888), 0x084f4100 },
+	{ _MMIO(0x9888), 0x0a4f0061 },
+	{ _MMIO(0x9888), 0x0c4f6c4c },
+	{ _MMIO(0x9888), 0x0e4f4b00 },
+	{ _MMIO(0x9888), 0x1a4f0000 },
+	{ _MMIO(0x9888), 0x1c4f0000 },
+	{ _MMIO(0x9888), 0x180f5000 },
+	{ _MMIO(0x9888), 0x1a0f8800 },
+	{ _MMIO(0x9888), 0x1c0f08a2 },
+	{ _MMIO(0x9888), 0x182c4000 },
+	{ _MMIO(0x9888), 0x1c2c1451 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c0010 },
+	{ _MMIO(0x9888), 0x01938000 },
+	{ _MMIO(0x9888), 0x0f938000 },
+	{ _MMIO(0x9888), 0x19938a28 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x19900177 },
+	{ _MMIO(0x9888), 0x1b900178 },
+	{ _MMIO(0x9888), 0x1d900125 },
+	{ _MMIO(0x9888), 0x1f900123 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x53901000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0c2e001f },
+	{ _MMIO(0x9888), 0x0a2f0000 },
+	{ _MMIO(0x9888), 0x10186800 },
+	{ _MMIO(0x9888), 0x11810019 },
+	{ _MMIO(0x9888), 0x15810013 },
+	{ _MMIO(0x9888), 0x13820020 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x17840000 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x21860000 },
+	{ _MMIO(0x9888), 0x178703e0 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x022e5400 },
+	{ _MMIO(0x9888), 0x002e0000 },
+	{ _MMIO(0x9888), 0x0e2e0080 },
+	{ _MMIO(0x9888), 0x082f0040 },
+	{ _MMIO(0x9888), 0x002f0000 },
+	{ _MMIO(0x9888), 0x06143000 },
+	{ _MMIO(0x9888), 0x06174000 },
+	{ _MMIO(0x9888), 0x06180012 },
+	{ _MMIO(0x9888), 0x00180000 },
+	{ _MMIO(0x9888), 0x0d804000 },
+	{ _MMIO(0x9888), 0x0f804000 },
+	{ _MMIO(0x9888), 0x05804000 },
+	{ _MMIO(0x9888), 0x09810200 },
+	{ _MMIO(0x9888), 0x0b810030 },
+	{ _MMIO(0x9888), 0x03810003 },
+	{ _MMIO(0x9888), 0x21819140 },
+	{ _MMIO(0x9888), 0x23819050 },
+	{ _MMIO(0x9888), 0x25810018 },
+	{ _MMIO(0x9888), 0x0b820980 },
+	{ _MMIO(0x9888), 0x03820d80 },
+	{ _MMIO(0x9888), 0x11820000 },
+	{ _MMIO(0x9888), 0x0182c000 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x09824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0d830004 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x0f831000 },
+	{ _MMIO(0x9888), 0x01848072 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x09844000 },
+	{ _MMIO(0x9888), 0x0f848000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x09860092 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x01869100 },
+	{ _MMIO(0x9888), 0x0f870065 },
+	{ _MMIO(0x9888), 0x01870000 },
+	{ _MMIO(0x9888), 0x19930800 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x1b952000 },
+	{ _MMIO(0x9888), 0x1d955055 },
+	{ _MMIO(0x9888), 0x1f951455 },
+	{ _MMIO(0x9888), 0x0992a000 },
+	{ _MMIO(0x9888), 0x0f928000 },
+	{ _MMIO(0x9888), 0x1192a800 },
+	{ _MMIO(0x9888), 0x1392028a },
+	{ _MMIO(0x9888), 0x0b92a000 },
+	{ _MMIO(0x9888), 0x0d922000 },
+	{ _MMIO(0x9888), 0x13908000 },
+	{ _MMIO(0x9888), 0x21908000 },
+	{ _MMIO(0x9888), 0x23908000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27908000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900c01 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900863 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900061 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x45900c22 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x19800343 },
+	{ _MMIO(0x9888), 0x39900340 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x41900003 },
+	{ _MMIO(0x9888), 0x03803180 },
+	{ _MMIO(0x9888), 0x058035e2 },
+	{ _MMIO(0x9888), 0x0780006a },
+	{ _MMIO(0x9888), 0x11800000 },
+	{ _MMIO(0x9888), 0x2181a000 },
+	{ _MMIO(0x9888), 0x2381000a },
+	{ _MMIO(0x9888), 0x1d950550 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d92a000 },
+	{ _MMIO(0x9888), 0x0f922000 },
+	{ _MMIO(0x9888), 0x13900170 },
+	{ _MMIO(0x9888), 0x21900171 },
+	{ _MMIO(0x9888), 0x23900172 },
+	{ _MMIO(0x9888), 0x25900173 },
+	{ _MMIO(0x9888), 0x27900174 },
+	{ _MMIO(0x9888), 0x29900175 },
+	{ _MMIO(0x9888), 0x2b900176 },
+	{ _MMIO(0x9888), 0x2d900177 },
+	{ _MMIO(0x9888), 0x2f90017f },
+	{ _MMIO(0x9888), 0x31900125 },
+	{ _MMIO(0x9888), 0x15900123 },
+	{ _MMIO(0x9888), 0x17900121 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43901084 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47901080 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49901084 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b901084 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900004 },
+	{ _MMIO(0x9888), 0x45900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x19800343 },
+	{ _MMIO(0x9888), 0x39900340 },
+	{ _MMIO(0x9888), 0x3f900000 },
+	{ _MMIO(0x9888), 0x41900080 },
+	{ _MMIO(0x9888), 0x03803180 },
+	{ _MMIO(0x9888), 0x058035e2 },
+	{ _MMIO(0x9888), 0x0780006a },
+	{ _MMIO(0x9888), 0x11800000 },
+	{ _MMIO(0x9888), 0x2181a000 },
+	{ _MMIO(0x9888), 0x2381000a },
+	{ _MMIO(0x9888), 0x1d950550 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d92a000 },
+	{ _MMIO(0x9888), 0x0f922000 },
+	{ _MMIO(0x9888), 0x13900180 },
+	{ _MMIO(0x9888), 0x21900181 },
+	{ _MMIO(0x9888), 0x23900182 },
+	{ _MMIO(0x9888), 0x25900183 },
+	{ _MMIO(0x9888), 0x27900184 },
+	{ _MMIO(0x9888), 0x29900185 },
+	{ _MMIO(0x9888), 0x2b900186 },
+	{ _MMIO(0x9888), 0x2d900187 },
+	{ _MMIO(0x9888), 0x2f900170 },
+	{ _MMIO(0x9888), 0x31900125 },
+	{ _MMIO(0x9888), 0x15900123 },
+	{ _MMIO(0x9888), 0x17900121 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43901084 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47901080 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49901084 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b901084 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900004 },
+	{ _MMIO(0x9888), 0x45900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x141c0160 },
+	{ _MMIO(0x9888), 0x161c0015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x002d5000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d5000 },
+	{ _MMIO(0x9888), 0x0a2d5000 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x0c2e5400 },
+	{ _MMIO(0x9888), 0x0e2e5515 },
+	{ _MMIO(0x9888), 0x102e0155 },
+	{ _MMIO(0x9888), 0x044cc000 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4cc000 },
+	{ _MMIO(0x9888), 0x0e4cc000 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x004ea000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084ea000 },
+	{ _MMIO(0x9888), 0x0a4ea000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x0e4f4b41 },
+	{ _MMIO(0x9888), 0x004f4200 },
+	{ _MMIO(0x9888), 0x024f404c },
+	{ _MMIO(0x9888), 0x1c4f0000 },
+	{ _MMIO(0x9888), 0x1a4f0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0a1bc000 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x001c0031 },
+	{ _MMIO(0x9888), 0x061c1900 },
+	{ _MMIO(0x9888), 0x081c1a33 },
+	{ _MMIO(0x9888), 0x0a1c1b35 },
+	{ _MMIO(0x9888), 0x0c1c3337 },
+	{ _MMIO(0x9888), 0x041c31c7 },
+	{ _MMIO(0x9888), 0x180f5000 },
+	{ _MMIO(0x9888), 0x1a0fa8aa },
+	{ _MMIO(0x9888), 0x1c0f0aaa },
+	{ _MMIO(0x9888), 0x182c8000 },
+	{ _MMIO(0x9888), 0x1c2c6aaa },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c2950 },
+	{ _MMIO(0x9888), 0x01938000 },
+	{ _MMIO(0x9888), 0x0f938000 },
+	{ _MMIO(0x9888), 0x1993aaaa },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29904000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900420 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900400 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x45900001 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c03b0 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f900c00 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x002d1000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x0c2e0400 },
+	{ _MMIO(0x9888), 0x0e2e1500 },
+	{ _MMIO(0x9888), 0x102e0140 },
+	{ _MMIO(0x9888), 0x044c4000 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4cc000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x004e2000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x1a4f4001 },
+	{ _MMIO(0x9888), 0x1c4f5005 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x180f1000 },
+	{ _MMIO(0x9888), 0x1a0fa800 },
+	{ _MMIO(0x9888), 0x1c0f0a00 },
+	{ _MMIO(0x9888), 0x182c4000 },
+	{ _MMIO(0x9888), 0x1c2c4015 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x03931980 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x01938000 },
+	{ _MMIO(0x9888), 0x0f938000 },
+	{ _MMIO(0x9888), 0x1993a00a },
+	{ _MMIO(0x9888), 0x07930000 },
+	{ _MMIO(0x9888), 0x09930000 },
+	{ _MMIO(0x9888), 0x1d900177 },
+	{ _MMIO(0x9888), 0x1f900178 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x53901000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x47900001 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x022d4000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0e2e0055 },
+	{ _MMIO(0x9888), 0x064c8000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x024f6100 },
+	{ _MMIO(0x9888), 0x044f416b },
+	{ _MMIO(0x9888), 0x064f004b },
+	{ _MMIO(0x9888), 0x1a4f0000 },
+	{ _MMIO(0x9888), 0x1a0f02a8 },
+	{ _MMIO(0x9888), 0x1a2c5500 },
+	{ _MMIO(0x9888), 0x0f808000 },
+	{ _MMIO(0x9888), 0x25810020 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x1f951000 },
+	{ _MMIO(0x9888), 0x13920200 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4d900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1_0_sku_gte_0x03[] = {
+	{ _MMIO(0x9888), 0x12643400 },
+	{ _MMIO(0x9888), 0x12653400 },
+	{ _MMIO(0x9888), 0x106c6800 },
+	{ _MMIO(0x9888), 0x126c001e },
+	{ _MMIO(0x9888), 0x166c0010 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e0154 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0055 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c4f5500 },
+	{ _MMIO(0x9888), 0x1a4f1554 },
+	{ _MMIO(0x9888), 0x0a640024 },
+	{ _MMIO(0x9888), 0x10640000 },
+	{ _MMIO(0x9888), 0x04640000 },
+	{ _MMIO(0x9888), 0x0c650024 },
+	{ _MMIO(0x9888), 0x10650000 },
+	{ _MMIO(0x9888), 0x06650000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0900 },
+	{ _MMIO(0x9888), 0x1c0f0aa0 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f02aa },
+	{ _MMIO(0x9888), 0x1c2c5400 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c5550 },
+	{ _MMIO(0x9888), 0x1993aa00 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900421 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900420 },
+	{ _MMIO(0x9888), 0x45900021 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1_0_sku_lt_0x03[] = {
+	{ _MMIO(0x9888), 0x14640340 },
+	{ _MMIO(0x9888), 0x14650340 },
+	{ _MMIO(0x9888), 0x106c6800 },
+	{ _MMIO(0x9888), 0x126c001e },
+	{ _MMIO(0x9888), 0x166c0010 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e0154 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0055 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c4f5500 },
+	{ _MMIO(0x9888), 0x1a4f1554 },
+	{ _MMIO(0x9888), 0x04642400 },
+	{ _MMIO(0x9888), 0x22640000 },
+	{ _MMIO(0x9888), 0x1a640000 },
+	{ _MMIO(0x9888), 0x06650024 },
+	{ _MMIO(0x9888), 0x22650000 },
+	{ _MMIO(0x9888), 0x1c650000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0900 },
+	{ _MMIO(0x9888), 0x1c0f0aa0 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f02aa },
+	{ _MMIO(0x9888), 0x1c2c5400 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c5550 },
+	{ _MMIO(0x9888), 0x1993aa00 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900421 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900420 },
+	{ _MMIO(0x9888), 0x45900021 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
+
+	if (dev_priv->drm.pdev->revision >= 0x03) {
+		regs[n] = mux_config_l3_1_0_sku_gte_0x03;
+		lens[n] = ARRAY_SIZE(mux_config_l3_1_0_sku_gte_0x03);
+		n++;
+	}
+	if (dev_priv->drm.pdev->revision < 0x03) {
+		regs[n] = mux_config_l3_1_0_sku_lt_0x03;
+		lens[n] = ARRAY_SIZE(mux_config_l3_1_0_sku_lt_0x03);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102d7800 },
+	{ _MMIO(0x9888), 0x122d79e0 },
+	{ _MMIO(0x9888), 0x0c2f0004 },
+	{ _MMIO(0x9888), 0x100e3800 },
+	{ _MMIO(0x9888), 0x180f0005 },
+	{ _MMIO(0x9888), 0x002d0940 },
+	{ _MMIO(0x9888), 0x022d802f },
+	{ _MMIO(0x9888), 0x042d4013 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0e2e0050 },
+	{ _MMIO(0x9888), 0x022f0010 },
+	{ _MMIO(0x9888), 0x002f0000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x040e0480 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x060f0027 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x1a0f0040 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x439014a0 },
+	{ _MMIO(0x9888), 0x459000a4 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x121300a0 },
+	{ _MMIO(0x9888), 0x141600ab },
+	{ _MMIO(0x9888), 0x123300a0 },
+	{ _MMIO(0x9888), 0x143600ab },
+	{ _MMIO(0x9888), 0x125300a0 },
+	{ _MMIO(0x9888), 0x145600ab },
+	{ _MMIO(0x9888), 0x0c2d4000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e01a0 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0065 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084c4000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044e2000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c0f0800 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f023f },
+	{ _MMIO(0x9888), 0x1e2c0003 },
+	{ _MMIO(0x9888), 0x1a2cc030 },
+	{ _MMIO(0x9888), 0x04132180 },
+	{ _MMIO(0x9888), 0x02130000 },
+	{ _MMIO(0x9888), 0x0c148000 },
+	{ _MMIO(0x9888), 0x0e142000 },
+	{ _MMIO(0x9888), 0x04148000 },
+	{ _MMIO(0x9888), 0x1e150140 },
+	{ _MMIO(0x9888), 0x1c150040 },
+	{ _MMIO(0x9888), 0x0c163000 },
+	{ _MMIO(0x9888), 0x0e160068 },
+	{ _MMIO(0x9888), 0x10160000 },
+	{ _MMIO(0x9888), 0x18160000 },
+	{ _MMIO(0x9888), 0x0a164000 },
+	{ _MMIO(0x9888), 0x04330043 },
+	{ _MMIO(0x9888), 0x02330000 },
+	{ _MMIO(0x9888), 0x0234a000 },
+	{ _MMIO(0x9888), 0x04342000 },
+	{ _MMIO(0x9888), 0x1c350015 },
+	{ _MMIO(0x9888), 0x02363460 },
+	{ _MMIO(0x9888), 0x10360000 },
+	{ _MMIO(0x9888), 0x04360000 },
+	{ _MMIO(0x9888), 0x06360000 },
+	{ _MMIO(0x9888), 0x08364000 },
+	{ _MMIO(0x9888), 0x06530043 },
+	{ _MMIO(0x9888), 0x02530000 },
+	{ _MMIO(0x9888), 0x0e548000 },
+	{ _MMIO(0x9888), 0x00548000 },
+	{ _MMIO(0x9888), 0x06542000 },
+	{ _MMIO(0x9888), 0x1e550400 },
+	{ _MMIO(0x9888), 0x1a552000 },
+	{ _MMIO(0x9888), 0x1c550100 },
+	{ _MMIO(0x9888), 0x0e563000 },
+	{ _MMIO(0x9888), 0x00563400 },
+	{ _MMIO(0x9888), 0x10560000 },
+	{ _MMIO(0x9888), 0x18560000 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x0c564000 },
+	{ _MMIO(0x9888), 0x1993a800 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b9014a0 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900820 },
+	{ _MMIO(0x9888), 0x45901022 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x141a0000 },
+	{ _MMIO(0x9888), 0x143a0000 },
+	{ _MMIO(0x9888), 0x145a0000 },
+	{ _MMIO(0x9888), 0x0c2d4000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e0150 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e006a },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064c4000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024e2000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c0f0bc0 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f0302 },
+	{ _MMIO(0x9888), 0x1e2c0003 },
+	{ _MMIO(0x9888), 0x1a2c00f0 },
+	{ _MMIO(0x9888), 0x021a3080 },
+	{ _MMIO(0x9888), 0x041a31e5 },
+	{ _MMIO(0x9888), 0x02148000 },
+	{ _MMIO(0x9888), 0x0414a000 },
+	{ _MMIO(0x9888), 0x1c150054 },
+	{ _MMIO(0x9888), 0x06168000 },
+	{ _MMIO(0x9888), 0x08168000 },
+	{ _MMIO(0x9888), 0x0a168000 },
+	{ _MMIO(0x9888), 0x0c3a3280 },
+	{ _MMIO(0x9888), 0x0e3a0063 },
+	{ _MMIO(0x9888), 0x063a0061 },
+	{ _MMIO(0x9888), 0x023a0000 },
+	{ _MMIO(0x9888), 0x0c348000 },
+	{ _MMIO(0x9888), 0x0e342000 },
+	{ _MMIO(0x9888), 0x06342000 },
+	{ _MMIO(0x9888), 0x1e350140 },
+	{ _MMIO(0x9888), 0x1c350100 },
+	{ _MMIO(0x9888), 0x18360028 },
+	{ _MMIO(0x9888), 0x0c368000 },
+	{ _MMIO(0x9888), 0x0e5a3080 },
+	{ _MMIO(0x9888), 0x005a3280 },
+	{ _MMIO(0x9888), 0x025a0063 },
+	{ _MMIO(0x9888), 0x0e548000 },
+	{ _MMIO(0x9888), 0x00548000 },
+	{ _MMIO(0x9888), 0x02542000 },
+	{ _MMIO(0x9888), 0x1e550400 },
+	{ _MMIO(0x9888), 0x1a552000 },
+	{ _MMIO(0x9888), 0x1c550001 },
+	{ _MMIO(0x9888), 0x18560080 },
+	{ _MMIO(0x9888), 0x02568000 },
+	{ _MMIO(0x9888), 0x04568000 },
+	{ _MMIO(0x9888), 0x1993a800 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x45901084 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900001 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x141a026b },
+	{ _MMIO(0x9888), 0x143a0173 },
+	{ _MMIO(0x9888), 0x145a026b },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0069 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x180f6000 },
+	{ _MMIO(0x9888), 0x1a0f030a },
+	{ _MMIO(0x9888), 0x1a2c03c0 },
+	{ _MMIO(0x9888), 0x041a37e7 },
+	{ _MMIO(0x9888), 0x021a0000 },
+	{ _MMIO(0x9888), 0x0414a000 },
+	{ _MMIO(0x9888), 0x1c150050 },
+	{ _MMIO(0x9888), 0x08168000 },
+	{ _MMIO(0x9888), 0x0a168000 },
+	{ _MMIO(0x9888), 0x003a3380 },
+	{ _MMIO(0x9888), 0x063a006f },
+	{ _MMIO(0x9888), 0x023a0000 },
+	{ _MMIO(0x9888), 0x00348000 },
+	{ _MMIO(0x9888), 0x06342000 },
+	{ _MMIO(0x9888), 0x1a352000 },
+	{ _MMIO(0x9888), 0x1c350100 },
+	{ _MMIO(0x9888), 0x02368000 },
+	{ _MMIO(0x9888), 0x0c368000 },
+	{ _MMIO(0x9888), 0x025a37e7 },
+	{ _MMIO(0x9888), 0x0254a000 },
+	{ _MMIO(0x9888), 0x1c550005 },
+	{ _MMIO(0x9888), 0x04568000 },
+	{ _MMIO(0x9888), 0x06568000 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900020 },
+	{ _MMIO(0x9888), 0x45901080 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+	{ _MMIO(0xe458), 0x00001000 },
+	{ _MMIO(0xe558), 0x00003002 },
+	{ _MMIO(0xe658), 0x00005004 },
+	{ _MMIO(0xe758), 0x00011010 },
+	{ _MMIO(0xe45c), 0x00050012 },
+	{ _MMIO(0xe55c), 0x00052051 },
+	{ _MMIO(0xe65c), 0x00000008 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x141a001f },
+	{ _MMIO(0x9888), 0x143a001f },
+	{ _MMIO(0x9888), 0x145a001f },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0e2e0094 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x1a0f00e0 },
+	{ _MMIO(0x9888), 0x1a2c0c00 },
+	{ _MMIO(0x9888), 0x061a0063 },
+	{ _MMIO(0x9888), 0x021a0000 },
+	{ _MMIO(0x9888), 0x06142000 },
+	{ _MMIO(0x9888), 0x1c150100 },
+	{ _MMIO(0x9888), 0x0c168000 },
+	{ _MMIO(0x9888), 0x043a3180 },
+	{ _MMIO(0x9888), 0x023a0000 },
+	{ _MMIO(0x9888), 0x04348000 },
+	{ _MMIO(0x9888), 0x1c350040 },
+	{ _MMIO(0x9888), 0x0a368000 },
+	{ _MMIO(0x9888), 0x045a0063 },
+	{ _MMIO(0x9888), 0x025a0000 },
+	{ _MMIO(0x9888), 0x04542000 },
+	{ _MMIO(0x9888), 0x1c550010 },
+	{ _MMIO(0x9888), 0x08568000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900004 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x19800000 },
+	{ _MMIO(0x9888), 0x07800063 },
+	{ _MMIO(0x9888), 0x11800000 },
+	{ _MMIO(0x9888), 0x23810008 },
+	{ _MMIO(0x9888), 0x1d950400 },
+	{ _MMIO(0x9888), 0x0f922000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_bxt(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "22b9519a-e9ba-4c41-8b54-f4f8ca14fa0a",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "012d72cf-82a9-4d25-8ddf-74076fd30797",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "ce416533-e49e-4211-80af-ec513590a914",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "398e2452-18d7-42d0-b241-e4d0a9148ada",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "d324a0d6-7269-4847-a5c2-6f71ddc7fed5",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "caf3596a-7bb1-4dec-b3b3-2a080d283b49",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "49b956e2-d5b9-47e0-9d8a-cee5e8cec527",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "f64ef50a-bdba-4b35-8f09-203c13d8ee5a",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "00ad5a41-7eab-4f7a-9103-49d411c67219",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "46dc44ca-491c-4cc1-a951-e7b3e62bf02b",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "8364e2a8-af63-40af-b0d5-42969a255654",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "175c8092-cb25-4d1e-8dc7-b4fdd39e2d92",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "d260f03f-b34d-4b49-a44e-436819117332",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "fa6ecf21-2cb8-4d0b-9308-6e4a7b4ca87a",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "5ee72f5c-092f-421e-8b70-225f7c3e9612",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_bxt(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_bxt(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.h b/drivers/gpu/drm/i915/i915_oa_bxt.h
new file mode 100644
index 000000000000..6cf7ba746e7e
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_bxt.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_BXT_H__
+#define __I915_OA_BXT_H__
+
+extern int i915_oa_n_builtin_metric_sets_bxt;
+
+extern int i915_oa_select_metric_set_bxt(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_bxt(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_bxt(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_chv.c b/drivers/gpu/drm/i915/i915_oa_chv.c
new file mode 100644
index 000000000000..aa6bece7e75f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_chv.c
@@ -0,0 +1,2873 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_chv.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_L3_4,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER_1,
+	METRIC_SET_ID_SAMPLER_2,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_chv = 14;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9888), 0x59800000 },
+	{ _MMIO(0x9888), 0x59800001 },
+	{ _MMIO(0x9888), 0x285a0006 },
+	{ _MMIO(0x9888), 0x2c110014 },
+	{ _MMIO(0x9888), 0x2e110000 },
+	{ _MMIO(0x9888), 0x2c310014 },
+	{ _MMIO(0x9888), 0x2e310000 },
+	{ _MMIO(0x9888), 0x2b8303df },
+	{ _MMIO(0x9888), 0x3580024f },
+	{ _MMIO(0x9888), 0x00580888 },
+	{ _MMIO(0x9888), 0x1e5a0015 },
+	{ _MMIO(0x9888), 0x205a0014 },
+	{ _MMIO(0x9888), 0x045a0000 },
+	{ _MMIO(0x9888), 0x025a0000 },
+	{ _MMIO(0x9888), 0x02180500 },
+	{ _MMIO(0x9888), 0x00190555 },
+	{ _MMIO(0x9888), 0x021d0500 },
+	{ _MMIO(0x9888), 0x021f0a00 },
+	{ _MMIO(0x9888), 0x00380444 },
+	{ _MMIO(0x9888), 0x02390500 },
+	{ _MMIO(0x9888), 0x003a0666 },
+	{ _MMIO(0x9888), 0x00100111 },
+	{ _MMIO(0x9888), 0x06110030 },
+	{ _MMIO(0x9888), 0x0a110031 },
+	{ _MMIO(0x9888), 0x0e110046 },
+	{ _MMIO(0x9888), 0x04110000 },
+	{ _MMIO(0x9888), 0x00110000 },
+	{ _MMIO(0x9888), 0x00130111 },
+	{ _MMIO(0x9888), 0x00300444 },
+	{ _MMIO(0x9888), 0x08310030 },
+	{ _MMIO(0x9888), 0x0c310031 },
+	{ _MMIO(0x9888), 0x10310046 },
+	{ _MMIO(0x9888), 0x04310000 },
+	{ _MMIO(0x9888), 0x00310000 },
+	{ _MMIO(0x9888), 0x00330444 },
+	{ _MMIO(0x9888), 0x038a0a00 },
+	{ _MMIO(0x9888), 0x018b0fff },
+	{ _MMIO(0x9888), 0x038b0a00 },
+	{ _MMIO(0x9888), 0x01855000 },
+	{ _MMIO(0x9888), 0x03850055 },
+	{ _MMIO(0x9888), 0x13830021 },
+	{ _MMIO(0x9888), 0x15830020 },
+	{ _MMIO(0x9888), 0x1783002f },
+	{ _MMIO(0x9888), 0x1983002e },
+	{ _MMIO(0x9888), 0x1b83002d },
+	{ _MMIO(0x9888), 0x1d83002c },
+	{ _MMIO(0x9888), 0x05830000 },
+	{ _MMIO(0x9888), 0x01840555 },
+	{ _MMIO(0x9888), 0x03840500 },
+	{ _MMIO(0x9888), 0x23800074 },
+	{ _MMIO(0x9888), 0x2580007d },
+	{ _MMIO(0x9888), 0x05800000 },
+	{ _MMIO(0x9888), 0x01805000 },
+	{ _MMIO(0x9888), 0x03800055 },
+	{ _MMIO(0x9888), 0x01865000 },
+	{ _MMIO(0x9888), 0x03860055 },
+	{ _MMIO(0x9888), 0x01875000 },
+	{ _MMIO(0x9888), 0x03870055 },
+	{ _MMIO(0x9888), 0x418000aa },
+	{ _MMIO(0x9888), 0x4380000a },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x4780000a },
+	{ _MMIO(0x9888), 0x49800000 },
+	{ _MMIO(0x9888), 0x4b800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x55800000 },
+	{ _MMIO(0x9888), 0x57800000 },
+	{ _MMIO(0x9888), 0x59800000 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x59800000 },
+	{ _MMIO(0x9888), 0x59800001 },
+	{ _MMIO(0x9888), 0x2e5800e0 },
+	{ _MMIO(0x9888), 0x2e3800e0 },
+	{ _MMIO(0x9888), 0x3580024f },
+	{ _MMIO(0x9888), 0x3d800140 },
+	{ _MMIO(0x9888), 0x08580042 },
+	{ _MMIO(0x9888), 0x0c580040 },
+	{ _MMIO(0x9888), 0x1058004c },
+	{ _MMIO(0x9888), 0x1458004b },
+	{ _MMIO(0x9888), 0x04580000 },
+	{ _MMIO(0x9888), 0x00580000 },
+	{ _MMIO(0x9888), 0x00195555 },
+	{ _MMIO(0x9888), 0x06380042 },
+	{ _MMIO(0x9888), 0x0a380040 },
+	{ _MMIO(0x9888), 0x0e38004c },
+	{ _MMIO(0x9888), 0x1238004b },
+	{ _MMIO(0x9888), 0x04380000 },
+	{ _MMIO(0x9888), 0x00384444 },
+	{ _MMIO(0x9888), 0x003a5555 },
+	{ _MMIO(0x9888), 0x018bffff },
+	{ _MMIO(0x9888), 0x01845555 },
+	{ _MMIO(0x9888), 0x17800074 },
+	{ _MMIO(0x9888), 0x1980007d },
+	{ _MMIO(0x9888), 0x1b80007c },
+	{ _MMIO(0x9888), 0x1d8000b6 },
+	{ _MMIO(0x9888), 0x1f8000b7 },
+	{ _MMIO(0x9888), 0x05800000 },
+	{ _MMIO(0x9888), 0x03800000 },
+	{ _MMIO(0x9888), 0x418000aa },
+	{ _MMIO(0x9888), 0x438000aa },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x47800000 },
+	{ _MMIO(0x9888), 0x4980012a },
+	{ _MMIO(0x9888), 0x4b80012a },
+	{ _MMIO(0x9888), 0x4d80012a },
+	{ _MMIO(0x9888), 0x4f80012a },
+	{ _MMIO(0x9888), 0x518001ce },
+	{ _MMIO(0x9888), 0x538001ce },
+	{ _MMIO(0x9888), 0x5580000e },
+	{ _MMIO(0x9888), 0x59800000 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x59800000 },
+	{ _MMIO(0x9888), 0x59800001 },
+	{ _MMIO(0x9888), 0x261e0000 },
+	{ _MMIO(0x9888), 0x281f000f },
+	{ _MMIO(0x9888), 0x2817001a },
+	{ _MMIO(0x9888), 0x2791001f },
+	{ _MMIO(0x9888), 0x27880019 },
+	{ _MMIO(0x9888), 0x2d890000 },
+	{ _MMIO(0x9888), 0x278a0007 },
+	{ _MMIO(0x9888), 0x298d001f },
+	{ _MMIO(0x9888), 0x278e0020 },
+	{ _MMIO(0x9888), 0x2b8f0012 },
+	{ _MMIO(0x9888), 0x29900000 },
+	{ _MMIO(0x9888), 0x00184000 },
+	{ _MMIO(0x9888), 0x02181000 },
+	{ _MMIO(0x9888), 0x02194000 },
+	{ _MMIO(0x9888), 0x141e0002 },
+	{ _MMIO(0x9888), 0x041e0000 },
+	{ _MMIO(0x9888), 0x001e0000 },
+	{ _MMIO(0x9888), 0x221f0015 },
+	{ _MMIO(0x9888), 0x041f0000 },
+	{ _MMIO(0x9888), 0x001f4000 },
+	{ _MMIO(0x9888), 0x021f0000 },
+	{ _MMIO(0x9888), 0x023a8000 },
+	{ _MMIO(0x9888), 0x0213c000 },
+	{ _MMIO(0x9888), 0x02164000 },
+	{ _MMIO(0x9888), 0x24170012 },
+	{ _MMIO(0x9888), 0x04170000 },
+	{ _MMIO(0x9888), 0x07910005 },
+	{ _MMIO(0x9888), 0x05910000 },
+	{ _MMIO(0x9888), 0x01911500 },
+	{ _MMIO(0x9888), 0x03910501 },
+	{ _MMIO(0x9888), 0x0d880002 },
+	{ _MMIO(0x9888), 0x1d880003 },
+	{ _MMIO(0x9888), 0x05880000 },
+	{ _MMIO(0x9888), 0x0b890032 },
+	{ _MMIO(0x9888), 0x1b890031 },
+	{ _MMIO(0x9888), 0x05890000 },
+	{ _MMIO(0x9888), 0x01890040 },
+	{ _MMIO(0x9888), 0x03890040 },
+	{ _MMIO(0x9888), 0x098a0000 },
+	{ _MMIO(0x9888), 0x198a0004 },
+	{ _MMIO(0x9888), 0x058a0000 },
+	{ _MMIO(0x9888), 0x018a8050 },
+	{ _MMIO(0x9888), 0x038a2050 },
+	{ _MMIO(0x9888), 0x018b95a9 },
+	{ _MMIO(0x9888), 0x038be5a9 },
+	{ _MMIO(0x9888), 0x018c1500 },
+	{ _MMIO(0x9888), 0x038c0501 },
+	{ _MMIO(0x9888), 0x178d0015 },
+	{ _MMIO(0x9888), 0x058d0000 },
+	{ _MMIO(0x9888), 0x138e0004 },
+	{ _MMIO(0x9888), 0x218e000c },
+	{ _MMIO(0x9888), 0x058e0000 },
+	{ _MMIO(0x9888), 0x018e0500 },
+	{ _MMIO(0x9888), 0x038e0101 },
+	{ _MMIO(0x9888), 0x0f8f0027 },
+	{ _MMIO(0x9888), 0x058f0000 },
+	{ _MMIO(0x9888), 0x018f0000 },
+	{ _MMIO(0x9888), 0x038f0001 },
+	{ _MMIO(0x9888), 0x11900013 },
+	{ _MMIO(0x9888), 0x1f900017 },
+	{ _MMIO(0x9888), 0x05900000 },
+	{ _MMIO(0x9888), 0x01900100 },
+	{ _MMIO(0x9888), 0x03900001 },
+	{ _MMIO(0x9888), 0x01845555 },
+	{ _MMIO(0x9888), 0x03845555 },
+	{ _MMIO(0x9888), 0x418000aa },
+	{ _MMIO(0x9888), 0x438000aa },
+	{ _MMIO(0x9888), 0x458000aa },
+	{ _MMIO(0x9888), 0x478000aa },
+	{ _MMIO(0x9888), 0x4980018c },
+	{ _MMIO(0x9888), 0x4b80014b },
+	{ _MMIO(0x9888), 0x4d800128 },
+	{ _MMIO(0x9888), 0x4f80012a },
+	{ _MMIO(0x9888), 0x51800187 },
+	{ _MMIO(0x9888), 0x5380014b },
+	{ _MMIO(0x9888), 0x55800149 },
+	{ _MMIO(0x9888), 0x5780010a },
+	{ _MMIO(0x9888), 0x59800000 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fff7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x105c0232 },
+	{ _MMIO(0x9888), 0x10580232 },
+	{ _MMIO(0x9888), 0x10380232 },
+	{ _MMIO(0x9888), 0x10dc0232 },
+	{ _MMIO(0x9888), 0x10d80232 },
+	{ _MMIO(0x9888), 0x10b80232 },
+	{ _MMIO(0x9888), 0x118e4400 },
+	{ _MMIO(0x9888), 0x025c6080 },
+	{ _MMIO(0x9888), 0x045c004b },
+	{ _MMIO(0x9888), 0x005c8000 },
+	{ _MMIO(0x9888), 0x00582080 },
+	{ _MMIO(0x9888), 0x0258004b },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00aa },
+	{ _MMIO(0x9888), 0x04386080 },
+	{ _MMIO(0x9888), 0x0638404b },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a380000 },
+	{ _MMIO(0x9888), 0x0c380000 },
+	{ _MMIO(0x9888), 0x00398000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x0cdc25c1 },
+	{ _MMIO(0x9888), 0x0adcc000 },
+	{ _MMIO(0x9888), 0x0ad825c1 },
+	{ _MMIO(0x9888), 0x18db4000 },
+	{ _MMIO(0x9888), 0x1adb0001 },
+	{ _MMIO(0x9888), 0x0e9f8000 },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x0eb825c1 },
+	{ _MMIO(0x9888), 0x18b80154 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x0d88c000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x258baa05 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x198c5400 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x098dc000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x098e05c0 },
+	{ _MMIO(0x9888), 0x058e0000 },
+	{ _MMIO(0x9888), 0x198f0020 },
+	{ _MMIO(0x9888), 0x2185aa0a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x19835000 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x19808000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x51800040 },
+	{ _MMIO(0x9888), 0x43800400 },
+	{ _MMIO(0x9888), 0x45800800 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800c62 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f801042 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x418014a4 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x10bf03da },
+	{ _MMIO(0x9888), 0x14bf0001 },
+	{ _MMIO(0x9888), 0x12980340 },
+	{ _MMIO(0x9888), 0x12990340 },
+	{ _MMIO(0x9888), 0x0cbf1187 },
+	{ _MMIO(0x9888), 0x0ebf1205 },
+	{ _MMIO(0x9888), 0x00bf0500 },
+	{ _MMIO(0x9888), 0x02bf042b },
+	{ _MMIO(0x9888), 0x04bf002c },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x0edac000 },
+	{ _MMIO(0x9888), 0x00da8000 },
+	{ _MMIO(0x9888), 0x02dac000 },
+	{ _MMIO(0x9888), 0x04da4000 },
+	{ _MMIO(0x9888), 0x04983400 },
+	{ _MMIO(0x9888), 0x10980000 },
+	{ _MMIO(0x9888), 0x06990034 },
+	{ _MMIO(0x9888), 0x10990000 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x0e9dc000 },
+	{ _MMIO(0x9888), 0x009d8000 },
+	{ _MMIO(0x9888), 0x029dc000 },
+	{ _MMIO(0x9888), 0x049d4000 },
+	{ _MMIO(0x9888), 0x109f02a8 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f00ba },
+	{ _MMIO(0x9888), 0x0cb88000 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x0eb95000 },
+	{ _MMIO(0x9888), 0x00b94000 },
+	{ _MMIO(0x9888), 0x02b95000 },
+	{ _MMIO(0x9888), 0x04b91000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x0cba4000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x258b800a },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b5500 },
+	{ _MMIO(0x9888), 0x198c4000 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x47800000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800060 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x103f03da },
+	{ _MMIO(0x9888), 0x143f0001 },
+	{ _MMIO(0x9888), 0x12180340 },
+	{ _MMIO(0x9888), 0x12190340 },
+	{ _MMIO(0x9888), 0x0c3f1187 },
+	{ _MMIO(0x9888), 0x0e3f1205 },
+	{ _MMIO(0x9888), 0x003f0500 },
+	{ _MMIO(0x9888), 0x023f042b },
+	{ _MMIO(0x9888), 0x043f002c },
+	{ _MMIO(0x9888), 0x0c5ac000 },
+	{ _MMIO(0x9888), 0x0e5ac000 },
+	{ _MMIO(0x9888), 0x005a8000 },
+	{ _MMIO(0x9888), 0x025ac000 },
+	{ _MMIO(0x9888), 0x045a4000 },
+	{ _MMIO(0x9888), 0x04183400 },
+	{ _MMIO(0x9888), 0x10180000 },
+	{ _MMIO(0x9888), 0x06190034 },
+	{ _MMIO(0x9888), 0x10190000 },
+	{ _MMIO(0x9888), 0x0c1dc000 },
+	{ _MMIO(0x9888), 0x0e1dc000 },
+	{ _MMIO(0x9888), 0x001d8000 },
+	{ _MMIO(0x9888), 0x021dc000 },
+	{ _MMIO(0x9888), 0x041d4000 },
+	{ _MMIO(0x9888), 0x101f02a8 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00ba },
+	{ _MMIO(0x9888), 0x0c388000 },
+	{ _MMIO(0x9888), 0x0c395000 },
+	{ _MMIO(0x9888), 0x0e395000 },
+	{ _MMIO(0x9888), 0x00394000 },
+	{ _MMIO(0x9888), 0x02395000 },
+	{ _MMIO(0x9888), 0x04391000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x0c3a4000 },
+	{ _MMIO(0x9888), 0x1b8aa800 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x258b4005 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800000 },
+	{ _MMIO(0x9888), 0x47800000 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800060 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x121b0340 },
+	{ _MMIO(0x9888), 0x103f0274 },
+	{ _MMIO(0x9888), 0x123f0000 },
+	{ _MMIO(0x9888), 0x129b0340 },
+	{ _MMIO(0x9888), 0x10bf0274 },
+	{ _MMIO(0x9888), 0x12bf0000 },
+	{ _MMIO(0x9888), 0x041b3400 },
+	{ _MMIO(0x9888), 0x101b0000 },
+	{ _MMIO(0x9888), 0x045c8000 },
+	{ _MMIO(0x9888), 0x0a3d4000 },
+	{ _MMIO(0x9888), 0x003f0080 },
+	{ _MMIO(0x9888), 0x023f0793 },
+	{ _MMIO(0x9888), 0x043f0014 },
+	{ _MMIO(0x9888), 0x04588000 },
+	{ _MMIO(0x9888), 0x005a8000 },
+	{ _MMIO(0x9888), 0x025ac000 },
+	{ _MMIO(0x9888), 0x045a4000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x001d8000 },
+	{ _MMIO(0x9888), 0x021dc000 },
+	{ _MMIO(0x9888), 0x041d4000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f002a },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x00394000 },
+	{ _MMIO(0x9888), 0x02395000 },
+	{ _MMIO(0x9888), 0x04399000 },
+	{ _MMIO(0x9888), 0x069b0034 },
+	{ _MMIO(0x9888), 0x109b0000 },
+	{ _MMIO(0x9888), 0x06dc4000 },
+	{ _MMIO(0x9888), 0x0cbd4000 },
+	{ _MMIO(0x9888), 0x0cbf0981 },
+	{ _MMIO(0x9888), 0x0ebf0a0f },
+	{ _MMIO(0x9888), 0x06d84000 },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x0edac000 },
+	{ _MMIO(0x9888), 0x0cdb4000 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x0e9dc000 },
+	{ _MMIO(0x9888), 0x109f02a8 },
+	{ _MMIO(0x9888), 0x0e9f0080 },
+	{ _MMIO(0x9888), 0x0cb84000 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x0eb95000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x258b8009 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x198c4000 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800c00 },
+	{ _MMIO(0x9888), 0x47800c63 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f8014a5 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800045 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_4[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_4[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_4[] = {
+	{ _MMIO(0x9888), 0x121a0340 },
+	{ _MMIO(0x9888), 0x103f0017 },
+	{ _MMIO(0x9888), 0x123f0020 },
+	{ _MMIO(0x9888), 0x129a0340 },
+	{ _MMIO(0x9888), 0x10bf0017 },
+	{ _MMIO(0x9888), 0x12bf0020 },
+	{ _MMIO(0x9888), 0x041a3400 },
+	{ _MMIO(0x9888), 0x101a0000 },
+	{ _MMIO(0x9888), 0x043b8000 },
+	{ _MMIO(0x9888), 0x0a3e0010 },
+	{ _MMIO(0x9888), 0x003f0200 },
+	{ _MMIO(0x9888), 0x023f0113 },
+	{ _MMIO(0x9888), 0x043f0014 },
+	{ _MMIO(0x9888), 0x02592000 },
+	{ _MMIO(0x9888), 0x005a8000 },
+	{ _MMIO(0x9888), 0x025ac000 },
+	{ _MMIO(0x9888), 0x045a4000 },
+	{ _MMIO(0x9888), 0x0a1c8000 },
+	{ _MMIO(0x9888), 0x001d8000 },
+	{ _MMIO(0x9888), 0x021dc000 },
+	{ _MMIO(0x9888), 0x041d4000 },
+	{ _MMIO(0x9888), 0x0a1e8000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f001a },
+	{ _MMIO(0x9888), 0x00394000 },
+	{ _MMIO(0x9888), 0x02395000 },
+	{ _MMIO(0x9888), 0x04391000 },
+	{ _MMIO(0x9888), 0x069a0034 },
+	{ _MMIO(0x9888), 0x109a0000 },
+	{ _MMIO(0x9888), 0x06bb4000 },
+	{ _MMIO(0x9888), 0x0abe0040 },
+	{ _MMIO(0x9888), 0x0cbf0984 },
+	{ _MMIO(0x9888), 0x0ebf0a02 },
+	{ _MMIO(0x9888), 0x02d94000 },
+	{ _MMIO(0x9888), 0x0cdac000 },
+	{ _MMIO(0x9888), 0x0edac000 },
+	{ _MMIO(0x9888), 0x0c9c0400 },
+	{ _MMIO(0x9888), 0x0c9dc000 },
+	{ _MMIO(0x9888), 0x0e9dc000 },
+	{ _MMIO(0x9888), 0x0c9e0400 },
+	{ _MMIO(0x9888), 0x109f02a8 },
+	{ _MMIO(0x9888), 0x0e9f0040 },
+	{ _MMIO(0x9888), 0x0cb95000 },
+	{ _MMIO(0x9888), 0x0eb95000 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x258b8009 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x198c4000 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185800a },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x1b830154 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x45800800 },
+	{ _MMIO(0x9888), 0x47800842 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f801084 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800044 },
+};
+
+static int
+get_l3_4_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_4;
+	lens[n] = ARRAY_SIZE(mux_config_l3_4);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00006000 },
+	{ _MMIO(0x2774), 0x0000f3ff },
+	{ _MMIO(0x2778), 0x00001800 },
+	{ _MMIO(0x277c), 0x0000fcff },
+	{ _MMIO(0x2780), 0x00000600 },
+	{ _MMIO(0x2784), 0x0000ff3f },
+	{ _MMIO(0x2788), 0x00000180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000060 },
+	{ _MMIO(0x2794), 0x0000fff3 },
+	{ _MMIO(0x2798), 0x00000018 },
+	{ _MMIO(0x279c), 0x0000fffc },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x143b000e },
+	{ _MMIO(0x9888), 0x043c55c0 },
+	{ _MMIO(0x9888), 0x0a1e0280 },
+	{ _MMIO(0x9888), 0x0c1e0408 },
+	{ _MMIO(0x9888), 0x10390000 },
+	{ _MMIO(0x9888), 0x12397a1f },
+	{ _MMIO(0x9888), 0x14bb000e },
+	{ _MMIO(0x9888), 0x04bc5000 },
+	{ _MMIO(0x9888), 0x0a9e0296 },
+	{ _MMIO(0x9888), 0x0c9e0008 },
+	{ _MMIO(0x9888), 0x10b90000 },
+	{ _MMIO(0x9888), 0x12b97a1f },
+	{ _MMIO(0x9888), 0x063b0042 },
+	{ _MMIO(0x9888), 0x103b0000 },
+	{ _MMIO(0x9888), 0x083c0000 },
+	{ _MMIO(0x9888), 0x0a3e0040 },
+	{ _MMIO(0x9888), 0x043f8000 },
+	{ _MMIO(0x9888), 0x02594000 },
+	{ _MMIO(0x9888), 0x045a8000 },
+	{ _MMIO(0x9888), 0x0c1c0400 },
+	{ _MMIO(0x9888), 0x041d8000 },
+	{ _MMIO(0x9888), 0x081e02c0 },
+	{ _MMIO(0x9888), 0x0e1e0000 },
+	{ _MMIO(0x9888), 0x0c1fa800 },
+	{ _MMIO(0x9888), 0x0e1f0260 },
+	{ _MMIO(0x9888), 0x101f0014 },
+	{ _MMIO(0x9888), 0x003905e0 },
+	{ _MMIO(0x9888), 0x06390bc0 },
+	{ _MMIO(0x9888), 0x02390018 },
+	{ _MMIO(0x9888), 0x04394000 },
+	{ _MMIO(0x9888), 0x04bb0042 },
+	{ _MMIO(0x9888), 0x10bb0000 },
+	{ _MMIO(0x9888), 0x02bc05c0 },
+	{ _MMIO(0x9888), 0x08bc0000 },
+	{ _MMIO(0x9888), 0x0abe0004 },
+	{ _MMIO(0x9888), 0x02bf8000 },
+	{ _MMIO(0x9888), 0x02d91000 },
+	{ _MMIO(0x9888), 0x02da8000 },
+	{ _MMIO(0x9888), 0x089c8000 },
+	{ _MMIO(0x9888), 0x029d8000 },
+	{ _MMIO(0x9888), 0x089e8000 },
+	{ _MMIO(0x9888), 0x0e9e0000 },
+	{ _MMIO(0x9888), 0x0e9fa806 },
+	{ _MMIO(0x9888), 0x109f0142 },
+	{ _MMIO(0x9888), 0x08b90617 },
+	{ _MMIO(0x9888), 0x0ab90be0 },
+	{ _MMIO(0x9888), 0x02b94000 },
+	{ _MMIO(0x9888), 0x0d88f000 },
+	{ _MMIO(0x9888), 0x0f88000c },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x018a8000 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x1b8a2800 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x238b52a0 },
+	{ _MMIO(0x9888), 0x258b6a95 },
+	{ _MMIO(0x9888), 0x278b0029 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c1500 },
+	{ _MMIO(0x9888), 0x1b8c0014 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x038d8000 },
+	{ _MMIO(0x9888), 0x058d2000 },
+	{ _MMIO(0x9888), 0x1f85aa80 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x01834000 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0184c000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1180c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x4d800444 },
+	{ _MMIO(0x9888), 0x3d800000 },
+	{ _MMIO(0x9888), 0x4f804000 },
+	{ _MMIO(0x9888), 0x43801080 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800084 },
+	{ _MMIO(0x9888), 0x53800044 },
+	{ _MMIO(0x9888), 0x47801080 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x3f800000 },
+	{ _MMIO(0x9888), 0x41800840 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler_1[] = {
+	{ _MMIO(0x9888), 0x18921400 },
+	{ _MMIO(0x9888), 0x149500ab },
+	{ _MMIO(0x9888), 0x18b21400 },
+	{ _MMIO(0x9888), 0x14b500ab },
+	{ _MMIO(0x9888), 0x18d21400 },
+	{ _MMIO(0x9888), 0x14d500ab },
+	{ _MMIO(0x9888), 0x0cdc8000 },
+	{ _MMIO(0x9888), 0x0edc4000 },
+	{ _MMIO(0x9888), 0x02dcc000 },
+	{ _MMIO(0x9888), 0x04dcc000 },
+	{ _MMIO(0x9888), 0x1abd00a0 },
+	{ _MMIO(0x9888), 0x0abd8000 },
+	{ _MMIO(0x9888), 0x0cd88000 },
+	{ _MMIO(0x9888), 0x0ed84000 },
+	{ _MMIO(0x9888), 0x04d88000 },
+	{ _MMIO(0x9888), 0x1adb0050 },
+	{ _MMIO(0x9888), 0x04db8000 },
+	{ _MMIO(0x9888), 0x06db8000 },
+	{ _MMIO(0x9888), 0x08db8000 },
+	{ _MMIO(0x9888), 0x0adb4000 },
+	{ _MMIO(0x9888), 0x109f02a0 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f00aa },
+	{ _MMIO(0x9888), 0x18b82500 },
+	{ _MMIO(0x9888), 0x02b88000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b84000 },
+	{ _MMIO(0x9888), 0x08b84000 },
+	{ _MMIO(0x9888), 0x0ab84000 },
+	{ _MMIO(0x9888), 0x0cb88000 },
+	{ _MMIO(0x9888), 0x0cb98000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x00b98000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x1aba0200 },
+	{ _MMIO(0x9888), 0x02ba8000 },
+	{ _MMIO(0x9888), 0x0cba8000 },
+	{ _MMIO(0x9888), 0x04908000 },
+	{ _MMIO(0x9888), 0x04918000 },
+	{ _MMIO(0x9888), 0x04927300 },
+	{ _MMIO(0x9888), 0x10920000 },
+	{ _MMIO(0x9888), 0x1893000a },
+	{ _MMIO(0x9888), 0x0a934000 },
+	{ _MMIO(0x9888), 0x0a946000 },
+	{ _MMIO(0x9888), 0x0c959000 },
+	{ _MMIO(0x9888), 0x0e950098 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x04b04000 },
+	{ _MMIO(0x9888), 0x04b14000 },
+	{ _MMIO(0x9888), 0x04b20073 },
+	{ _MMIO(0x9888), 0x10b20000 },
+	{ _MMIO(0x9888), 0x04b38000 },
+	{ _MMIO(0x9888), 0x06b38000 },
+	{ _MMIO(0x9888), 0x08b34000 },
+	{ _MMIO(0x9888), 0x04b4c000 },
+	{ _MMIO(0x9888), 0x02b59890 },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x06d04000 },
+	{ _MMIO(0x9888), 0x06d14000 },
+	{ _MMIO(0x9888), 0x06d20073 },
+	{ _MMIO(0x9888), 0x10d20000 },
+	{ _MMIO(0x9888), 0x18d30020 },
+	{ _MMIO(0x9888), 0x02d38000 },
+	{ _MMIO(0x9888), 0x0cd34000 },
+	{ _MMIO(0x9888), 0x0ad48000 },
+	{ _MMIO(0x9888), 0x04d42000 },
+	{ _MMIO(0x9888), 0x0ed59000 },
+	{ _MMIO(0x9888), 0x00d59800 },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x0f88000e },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b5500 },
+	{ _MMIO(0x9888), 0x258b000a },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x0d8d8000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x2185000a },
+	{ _MMIO(0x9888), 0x1b830150 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d848000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d808000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47801021 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800c64 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800c02 },
+};
+
+static int
+get_sampler_1_mux_config(struct drm_i915_private *dev_priv,
+			 const struct i915_oa_reg **regs,
+			 int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler_1;
+	lens[n] = ARRAY_SIZE(mux_config_sampler_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler_2[] = {
+	{ _MMIO(0x9888), 0x18121400 },
+	{ _MMIO(0x9888), 0x141500ab },
+	{ _MMIO(0x9888), 0x18321400 },
+	{ _MMIO(0x9888), 0x143500ab },
+	{ _MMIO(0x9888), 0x18521400 },
+	{ _MMIO(0x9888), 0x145500ab },
+	{ _MMIO(0x9888), 0x0c5c8000 },
+	{ _MMIO(0x9888), 0x0e5c4000 },
+	{ _MMIO(0x9888), 0x025cc000 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x1a3d00a0 },
+	{ _MMIO(0x9888), 0x0a3d8000 },
+	{ _MMIO(0x9888), 0x0c588000 },
+	{ _MMIO(0x9888), 0x0e584000 },
+	{ _MMIO(0x9888), 0x04588000 },
+	{ _MMIO(0x9888), 0x1a5b0050 },
+	{ _MMIO(0x9888), 0x045b8000 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b8000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x101f02a0 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00aa },
+	{ _MMIO(0x9888), 0x18382500 },
+	{ _MMIO(0x9888), 0x02388000 },
+	{ _MMIO(0x9888), 0x04384000 },
+	{ _MMIO(0x9888), 0x06384000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c388000 },
+	{ _MMIO(0x9888), 0x0c398000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x00398000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x1a3a0200 },
+	{ _MMIO(0x9888), 0x023a8000 },
+	{ _MMIO(0x9888), 0x0c3a8000 },
+	{ _MMIO(0x9888), 0x04108000 },
+	{ _MMIO(0x9888), 0x04118000 },
+	{ _MMIO(0x9888), 0x04127300 },
+	{ _MMIO(0x9888), 0x10120000 },
+	{ _MMIO(0x9888), 0x1813000a },
+	{ _MMIO(0x9888), 0x0a134000 },
+	{ _MMIO(0x9888), 0x0a146000 },
+	{ _MMIO(0x9888), 0x0c159000 },
+	{ _MMIO(0x9888), 0x0e150098 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04304000 },
+	{ _MMIO(0x9888), 0x04314000 },
+	{ _MMIO(0x9888), 0x04320073 },
+	{ _MMIO(0x9888), 0x10320000 },
+	{ _MMIO(0x9888), 0x04338000 },
+	{ _MMIO(0x9888), 0x06338000 },
+	{ _MMIO(0x9888), 0x08334000 },
+	{ _MMIO(0x9888), 0x0434c000 },
+	{ _MMIO(0x9888), 0x02359890 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x06504000 },
+	{ _MMIO(0x9888), 0x06514000 },
+	{ _MMIO(0x9888), 0x06520073 },
+	{ _MMIO(0x9888), 0x10520000 },
+	{ _MMIO(0x9888), 0x18530020 },
+	{ _MMIO(0x9888), 0x02538000 },
+	{ _MMIO(0x9888), 0x0c534000 },
+	{ _MMIO(0x9888), 0x0a548000 },
+	{ _MMIO(0x9888), 0x04542000 },
+	{ _MMIO(0x9888), 0x0e559000 },
+	{ _MMIO(0x9888), 0x00559800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x1b8aa000 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x258b0005 },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x2185000a },
+	{ _MMIO(0x9888), 0x1b830150 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0d848000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x07844000 },
+	{ _MMIO(0x9888), 0x1d808000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x17804000 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47801021 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800c64 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x41800c02 },
+};
+
+static int
+get_sampler_2_mux_config(struct drm_i915_private *dev_priv,
+			 const struct i915_oa_reg **regs,
+			 int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler_2;
+	lens[n] = ARRAY_SIZE(mux_config_sampler_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x0000fe7f },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000ffbf },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fff7 },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fff9 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x16154d60 },
+	{ _MMIO(0x9888), 0x16352e60 },
+	{ _MMIO(0x9888), 0x16554d60 },
+	{ _MMIO(0x9888), 0x16950000 },
+	{ _MMIO(0x9888), 0x16b50000 },
+	{ _MMIO(0x9888), 0x16d50000 },
+	{ _MMIO(0x9888), 0x005c8000 },
+	{ _MMIO(0x9888), 0x045cc000 },
+	{ _MMIO(0x9888), 0x065c4000 },
+	{ _MMIO(0x9888), 0x083d8000 },
+	{ _MMIO(0x9888), 0x0a3d8000 },
+	{ _MMIO(0x9888), 0x0458c000 },
+	{ _MMIO(0x9888), 0x025b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0c1fa000 },
+	{ _MMIO(0x9888), 0x0e1f00aa },
+	{ _MMIO(0x9888), 0x02384000 },
+	{ _MMIO(0x9888), 0x04388000 },
+	{ _MMIO(0x9888), 0x06388000 },
+	{ _MMIO(0x9888), 0x08384000 },
+	{ _MMIO(0x9888), 0x0a384000 },
+	{ _MMIO(0x9888), 0x0c384000 },
+	{ _MMIO(0x9888), 0x00398000 },
+	{ _MMIO(0x9888), 0x0239a000 },
+	{ _MMIO(0x9888), 0x0439a000 },
+	{ _MMIO(0x9888), 0x06392000 },
+	{ _MMIO(0x9888), 0x043a8000 },
+	{ _MMIO(0x9888), 0x063a8000 },
+	{ _MMIO(0x9888), 0x08138000 },
+	{ _MMIO(0x9888), 0x0a138000 },
+	{ _MMIO(0x9888), 0x06143000 },
+	{ _MMIO(0x9888), 0x0415cfc7 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x02338000 },
+	{ _MMIO(0x9888), 0x0c338000 },
+	{ _MMIO(0x9888), 0x04342000 },
+	{ _MMIO(0x9888), 0x06344000 },
+	{ _MMIO(0x9888), 0x0035c700 },
+	{ _MMIO(0x9888), 0x063500cf },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04538000 },
+	{ _MMIO(0x9888), 0x06538000 },
+	{ _MMIO(0x9888), 0x0454c000 },
+	{ _MMIO(0x9888), 0x0255cfc7 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06dc8000 },
+	{ _MMIO(0x9888), 0x08dc4000 },
+	{ _MMIO(0x9888), 0x0cdcc000 },
+	{ _MMIO(0x9888), 0x0edcc000 },
+	{ _MMIO(0x9888), 0x1abd00a8 },
+	{ _MMIO(0x9888), 0x0cd8c000 },
+	{ _MMIO(0x9888), 0x0ed84000 },
+	{ _MMIO(0x9888), 0x0edb8000 },
+	{ _MMIO(0x9888), 0x18db0800 },
+	{ _MMIO(0x9888), 0x1adb0254 },
+	{ _MMIO(0x9888), 0x0e9faa00 },
+	{ _MMIO(0x9888), 0x109f02aa },
+	{ _MMIO(0x9888), 0x0eb84000 },
+	{ _MMIO(0x9888), 0x16b84000 },
+	{ _MMIO(0x9888), 0x18b8156a },
+	{ _MMIO(0x9888), 0x06b98000 },
+	{ _MMIO(0x9888), 0x08b9a000 },
+	{ _MMIO(0x9888), 0x0ab9a000 },
+	{ _MMIO(0x9888), 0x0cb9a000 },
+	{ _MMIO(0x9888), 0x0eb9a000 },
+	{ _MMIO(0x9888), 0x18baa000 },
+	{ _MMIO(0x9888), 0x1aba0002 },
+	{ _MMIO(0x9888), 0x16934000 },
+	{ _MMIO(0x9888), 0x1893000a },
+	{ _MMIO(0x9888), 0x0a947000 },
+	{ _MMIO(0x9888), 0x0c95c5c1 },
+	{ _MMIO(0x9888), 0x0e9500c3 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x0eb38000 },
+	{ _MMIO(0x9888), 0x16b30040 },
+	{ _MMIO(0x9888), 0x18b30020 },
+	{ _MMIO(0x9888), 0x06b48000 },
+	{ _MMIO(0x9888), 0x08b41000 },
+	{ _MMIO(0x9888), 0x0ab48000 },
+	{ _MMIO(0x9888), 0x06b5c500 },
+	{ _MMIO(0x9888), 0x08b500c3 },
+	{ _MMIO(0x9888), 0x0eb5c100 },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x16d31500 },
+	{ _MMIO(0x9888), 0x08d4e000 },
+	{ _MMIO(0x9888), 0x08d5c100 },
+	{ _MMIO(0x9888), 0x0ad5c3c5 },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x0d88f800 },
+	{ _MMIO(0x9888), 0x0f88000f },
+	{ _MMIO(0x9888), 0x038a8000 },
+	{ _MMIO(0x9888), 0x058a8000 },
+	{ _MMIO(0x9888), 0x078a8000 },
+	{ _MMIO(0x9888), 0x098a8000 },
+	{ _MMIO(0x9888), 0x0b8a8000 },
+	{ _MMIO(0x9888), 0x0d8a8000 },
+	{ _MMIO(0x9888), 0x258baaa5 },
+	{ _MMIO(0x9888), 0x278b002a },
+	{ _MMIO(0x9888), 0x238b2a80 },
+	{ _MMIO(0x9888), 0x0f8c4000 },
+	{ _MMIO(0x9888), 0x178c2000 },
+	{ _MMIO(0x9888), 0x198c5500 },
+	{ _MMIO(0x9888), 0x1b8c0015 },
+	{ _MMIO(0x9888), 0x078d8000 },
+	{ _MMIO(0x9888), 0x098da000 },
+	{ _MMIO(0x9888), 0x0b8da000 },
+	{ _MMIO(0x9888), 0x0d8da000 },
+	{ _MMIO(0x9888), 0x0f8da000 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800c42 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45800063 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x47800800 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f8014a4 },
+	{ _MMIO(0x9888), 0x41801042 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x0000fe7f },
+	{ _MMIO(0x2780), 0x00000000 },
+	{ _MMIO(0x2784), 0x0000ff9f },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000ffe7 },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fffb },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000fffd },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x16150000 },
+	{ _MMIO(0x9888), 0x16350000 },
+	{ _MMIO(0x9888), 0x16550000 },
+	{ _MMIO(0x9888), 0x16952e60 },
+	{ _MMIO(0x9888), 0x16b54d60 },
+	{ _MMIO(0x9888), 0x16d52e60 },
+	{ _MMIO(0x9888), 0x065c8000 },
+	{ _MMIO(0x9888), 0x085cc000 },
+	{ _MMIO(0x9888), 0x0a5cc000 },
+	{ _MMIO(0x9888), 0x0c5c4000 },
+	{ _MMIO(0x9888), 0x0e3d8000 },
+	{ _MMIO(0x9888), 0x183da000 },
+	{ _MMIO(0x9888), 0x06588000 },
+	{ _MMIO(0x9888), 0x08588000 },
+	{ _MMIO(0x9888), 0x0a584000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x185b5800 },
+	{ _MMIO(0x9888), 0x1a5b000a },
+	{ _MMIO(0x9888), 0x0e1faa00 },
+	{ _MMIO(0x9888), 0x101f02aa },
+	{ _MMIO(0x9888), 0x0e384000 },
+	{ _MMIO(0x9888), 0x16384000 },
+	{ _MMIO(0x9888), 0x18382a55 },
+	{ _MMIO(0x9888), 0x06398000 },
+	{ _MMIO(0x9888), 0x0839a000 },
+	{ _MMIO(0x9888), 0x0a39a000 },
+	{ _MMIO(0x9888), 0x0c39a000 },
+	{ _MMIO(0x9888), 0x0e39a000 },
+	{ _MMIO(0x9888), 0x1a3a02a0 },
+	{ _MMIO(0x9888), 0x0e138000 },
+	{ _MMIO(0x9888), 0x16130500 },
+	{ _MMIO(0x9888), 0x06148000 },
+	{ _MMIO(0x9888), 0x08146000 },
+	{ _MMIO(0x9888), 0x0615c100 },
+	{ _MMIO(0x9888), 0x0815c500 },
+	{ _MMIO(0x9888), 0x0a1500c3 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x16335040 },
+	{ _MMIO(0x9888), 0x08349000 },
+	{ _MMIO(0x9888), 0x0a341000 },
+	{ _MMIO(0x9888), 0x083500c1 },
+	{ _MMIO(0x9888), 0x0a35c500 },
+	{ _MMIO(0x9888), 0x0c3500c3 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x1853002a },
+	{ _MMIO(0x9888), 0x0a54e000 },
+	{ _MMIO(0x9888), 0x0c55c500 },
+	{ _MMIO(0x9888), 0x0e55c1c3 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x00dc8000 },
+	{ _MMIO(0x9888), 0x02dcc000 },
+	{ _MMIO(0x9888), 0x04dc4000 },
+	{ _MMIO(0x9888), 0x04bd8000 },
+	{ _MMIO(0x9888), 0x06bd8000 },
+	{ _MMIO(0x9888), 0x02d8c000 },
+	{ _MMIO(0x9888), 0x02db8000 },
+	{ _MMIO(0x9888), 0x04db4000 },
+	{ _MMIO(0x9888), 0x06db4000 },
+	{ _MMIO(0x9888), 0x08db8000 },
+	{ _MMIO(0x9888), 0x0c9fa000 },
+	{ _MMIO(0x9888), 0x0e9f00aa },
+	{ _MMIO(0x9888), 0x02b84000 },
+	{ _MMIO(0x9888), 0x04b84000 },
+	{ _MMIO(0x9888), 0x06b84000 },
+	{ _MMIO(0x9888), 0x08b84000 },
+	{ _MMIO(0x9888), 0x0ab88000 },
+	{ _MMIO(0x9888), 0x0cb88000 },
+	{ _MMIO(0x9888), 0x00b98000 },
+	{ _MMIO(0x9888), 0x02b9a000 },
+	{ _MMIO(0x9888), 0x04b9a000 },
+	{ _MMIO(0x9888), 0x06b92000 },
+	{ _MMIO(0x9888), 0x0aba8000 },
+	{ _MMIO(0x9888), 0x0cba8000 },
+	{ _MMIO(0x9888), 0x04938000 },
+	{ _MMIO(0x9888), 0x06938000 },
+	{ _MMIO(0x9888), 0x0494c000 },
+	{ _MMIO(0x9888), 0x0295cfc7 },
+	{ _MMIO(0x9888), 0x10950000 },
+	{ _MMIO(0x9888), 0x02b38000 },
+	{ _MMIO(0x9888), 0x08b38000 },
+	{ _MMIO(0x9888), 0x04b42000 },
+	{ _MMIO(0x9888), 0x06b41000 },
+	{ _MMIO(0x9888), 0x00b5c700 },
+	{ _MMIO(0x9888), 0x04b500cf },
+	{ _MMIO(0x9888), 0x10b50000 },
+	{ _MMIO(0x9888), 0x0ad38000 },
+	{ _MMIO(0x9888), 0x0cd38000 },
+	{ _MMIO(0x9888), 0x06d46000 },
+	{ _MMIO(0x9888), 0x04d5c700 },
+	{ _MMIO(0x9888), 0x06d500cf },
+	{ _MMIO(0x9888), 0x10d50000 },
+	{ _MMIO(0x9888), 0x03888000 },
+	{ _MMIO(0x9888), 0x05888000 },
+	{ _MMIO(0x9888), 0x07888000 },
+	{ _MMIO(0x9888), 0x09888000 },
+	{ _MMIO(0x9888), 0x0b888000 },
+	{ _MMIO(0x9888), 0x0d880400 },
+	{ _MMIO(0x9888), 0x0f8a8000 },
+	{ _MMIO(0x9888), 0x198a8000 },
+	{ _MMIO(0x9888), 0x1b8aaaa0 },
+	{ _MMIO(0x9888), 0x1d8a0002 },
+	{ _MMIO(0x9888), 0x258b555a },
+	{ _MMIO(0x9888), 0x278b0015 },
+	{ _MMIO(0x9888), 0x238b5500 },
+	{ _MMIO(0x9888), 0x038c4000 },
+	{ _MMIO(0x9888), 0x058c4000 },
+	{ _MMIO(0x9888), 0x078c4000 },
+	{ _MMIO(0x9888), 0x098c4000 },
+	{ _MMIO(0x9888), 0x0b8c4000 },
+	{ _MMIO(0x9888), 0x0d8c4000 },
+	{ _MMIO(0x9888), 0x018d8000 },
+	{ _MMIO(0x9888), 0x038da000 },
+	{ _MMIO(0x9888), 0x058da000 },
+	{ _MMIO(0x9888), 0x078d2000 },
+	{ _MMIO(0x9888), 0x2185aaaa },
+	{ _MMIO(0x9888), 0x2385002a },
+	{ _MMIO(0x9888), 0x1f85aa00 },
+	{ _MMIO(0x9888), 0x0f834000 },
+	{ _MMIO(0x9888), 0x19835400 },
+	{ _MMIO(0x9888), 0x1b830155 },
+	{ _MMIO(0x9888), 0x03834000 },
+	{ _MMIO(0x9888), 0x05834000 },
+	{ _MMIO(0x9888), 0x07834000 },
+	{ _MMIO(0x9888), 0x09834000 },
+	{ _MMIO(0x9888), 0x0b834000 },
+	{ _MMIO(0x9888), 0x0d834000 },
+	{ _MMIO(0x9888), 0x0784c000 },
+	{ _MMIO(0x9888), 0x0984c000 },
+	{ _MMIO(0x9888), 0x0b84c000 },
+	{ _MMIO(0x9888), 0x0d84c000 },
+	{ _MMIO(0x9888), 0x0f84c000 },
+	{ _MMIO(0x9888), 0x01848000 },
+	{ _MMIO(0x9888), 0x0384c000 },
+	{ _MMIO(0x9888), 0x0584c000 },
+	{ _MMIO(0x9888), 0x1780c000 },
+	{ _MMIO(0x9888), 0x1980c000 },
+	{ _MMIO(0x9888), 0x1b80c000 },
+	{ _MMIO(0x9888), 0x1d80c000 },
+	{ _MMIO(0x9888), 0x1f80c000 },
+	{ _MMIO(0x9888), 0x11808000 },
+	{ _MMIO(0x9888), 0x1380c000 },
+	{ _MMIO(0x9888), 0x1580c000 },
+	{ _MMIO(0x9888), 0x4f800000 },
+	{ _MMIO(0x9888), 0x43800882 },
+	{ _MMIO(0x9888), 0x51800000 },
+	{ _MMIO(0x9888), 0x45801082 },
+	{ _MMIO(0x9888), 0x53800000 },
+	{ _MMIO(0x9888), 0x478014a5 },
+	{ _MMIO(0x9888), 0x21800000 },
+	{ _MMIO(0x9888), 0x31800000 },
+	{ _MMIO(0x9888), 0x4d800000 },
+	{ _MMIO(0x9888), 0x3f800002 },
+	{ _MMIO(0x9888), 0x41800c62 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x59800000 },
+	{ _MMIO(0x9888), 0x59800001 },
+	{ _MMIO(0x9888), 0x338b0000 },
+	{ _MMIO(0x9888), 0x258b0066 },
+	{ _MMIO(0x9888), 0x058b0000 },
+	{ _MMIO(0x9888), 0x038b0000 },
+	{ _MMIO(0x9888), 0x03844000 },
+	{ _MMIO(0x9888), 0x47800080 },
+	{ _MMIO(0x9888), 0x57800000 },
+	{ _MMIO(0x1823a4), 0x00000000 },
+	{ _MMIO(0x9888), 0x59800000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_chv(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_L3_4:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_4_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_4\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_4;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_4);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_4;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_4);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_1_mux_config(dev_priv,
+						 dev_priv->perf.oa.mux_regs,
+						 dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler_1);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_2_mux_config(dev_priv,
+						 dev_priv->perf.oa.mux_regs,
+						 dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler_2);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "9d8a3af5-c02c-4a4a-b947-f1672469e0fb",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "f522a89c-ecd1-4522-8331-3383c54af5f5",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "a9ccc03d-a943-4e6b-9cd6-13e063075927",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "2cf0c064-68df-4fac-9b3f-57f51ca8a069",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "78a87ff9-543a-49ce-95ea-26d86071ea93",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "9f2cece5-7bfe-4320-ad66-8c7cc526bec5",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "d890ef38-d309-47e4-b8b5-aa779bb19ab0",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_l3_4_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_4);
+}
+
+static struct device_attribute dev_attr_l3_4_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_4_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_4[] = {
+	&dev_attr_l3_4_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_4 = {
+	.name = "5fdff4a6-9dc8-45e1-bfda-ef54869fbdd4",
+	.attrs =  attrs_l3_4,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "2c0e45e1-7e2c-4a14-ae00-0b7ec868b8aa",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_1);
+}
+
+static struct device_attribute dev_attr_sampler_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler_1[] = {
+	&dev_attr_sampler_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler_1 = {
+	.name = "71148d78-baf5-474f-878a-e23158d0265d",
+	.attrs =  attrs_sampler_1,
+};
+
+static ssize_t
+show_sampler_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_2);
+}
+
+static struct device_attribute dev_attr_sampler_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler_2[] = {
+	&dev_attr_sampler_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler_2 = {
+	.name = "b996a2b7-c59c-492d-877a-8cd54fd6df84",
+	.attrs =  attrs_sampler_2,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "eb2fecba-b431-42e7-8261-fe9429a6e67a",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "60749470-a648-4a4b-9f10-dbfe1e36e44d",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "4a534b07-cba3-414d-8d60-874830e883aa",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_chv(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_4);
+		if (ret)
+			goto error_l3_4;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
+		if (ret)
+			goto error_sampler_1;
+	}
+	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
+		if (ret)
+			goto error_sampler_2;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
+error_sampler_2:
+	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
+error_sampler_1:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
+error_l3_4:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_chv(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
+	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_chv.h b/drivers/gpu/drm/i915/i915_oa_chv.h
new file mode 100644
index 000000000000..8b8bdc26d726
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_chv.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_CHV_H__
+#define __I915_OA_CHV_H__
+
+extern int i915_oa_n_builtin_metric_sets_chv;
+
+extern int i915_oa_select_metric_set_chv(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_chv(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_chv(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_glk.c b/drivers/gpu/drm/i915/i915_oa_glk.c
new file mode 100644
index 000000000000..2f356d51bff8
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_glk.c
@@ -0,0 +1,2602 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_glk.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_glk = 15;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9888), 0x166c00f0 },
+	{ _MMIO(0x9888), 0x12120280 },
+	{ _MMIO(0x9888), 0x12320280 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900c00 },
+	{ _MMIO(0x9888), 0x419000a0 },
+	{ _MMIO(0x9888), 0x002d1000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d5000 },
+	{ _MMIO(0x9888), 0x0a2d1000 },
+	{ _MMIO(0x9888), 0x0c2e0800 },
+	{ _MMIO(0x9888), 0x0e2e5900 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4c8000 },
+	{ _MMIO(0x9888), 0x0e4c4000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e2000 },
+	{ _MMIO(0x9888), 0x1c4f0010 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1a0fcc00 },
+	{ _MMIO(0x9888), 0x1c0f0002 },
+	{ _MMIO(0x9888), 0x1c2c0040 },
+	{ _MMIO(0x9888), 0x00101000 },
+	{ _MMIO(0x9888), 0x04101000 },
+	{ _MMIO(0x9888), 0x00114000 },
+	{ _MMIO(0x9888), 0x08114000 },
+	{ _MMIO(0x9888), 0x00120020 },
+	{ _MMIO(0x9888), 0x08120021 },
+	{ _MMIO(0x9888), 0x00141000 },
+	{ _MMIO(0x9888), 0x08141000 },
+	{ _MMIO(0x9888), 0x02308000 },
+	{ _MMIO(0x9888), 0x04302000 },
+	{ _MMIO(0x9888), 0x06318000 },
+	{ _MMIO(0x9888), 0x08318000 },
+	{ _MMIO(0x9888), 0x06320800 },
+	{ _MMIO(0x9888), 0x08320840 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x06344000 },
+	{ _MMIO(0x9888), 0x08344000 },
+	{ _MMIO(0x9888), 0x0d931831 },
+	{ _MMIO(0x9888), 0x0f939f3f },
+	{ _MMIO(0x9888), 0x01939e80 },
+	{ _MMIO(0x9888), 0x039303bc },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1993002a },
+	{ _MMIO(0x9888), 0x07930000 },
+	{ _MMIO(0x9888), 0x09930000 },
+	{ _MMIO(0x9888), 0x1d900177 },
+	{ _MMIO(0x9888), 0x1f900187 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x53901110 },
+	{ _MMIO(0x9888), 0x43900423 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x47900c02 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900020 },
+	{ _MMIO(0x9888), 0x59901111 },
+	{ _MMIO(0x9888), 0x4b900421 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x45900821 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x39900340 },
+	{ _MMIO(0x9888), 0x3f900c00 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x002d5000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d4000 },
+	{ _MMIO(0x9888), 0x0a2d1000 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x0c2e1400 },
+	{ _MMIO(0x9888), 0x0e2e5100 },
+	{ _MMIO(0x9888), 0x102e0114 },
+	{ _MMIO(0x9888), 0x044cc000 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4c8000 },
+	{ _MMIO(0x9888), 0x0e4c4000 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x004ea000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e2000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x004f6b42 },
+	{ _MMIO(0x9888), 0x064f6200 },
+	{ _MMIO(0x9888), 0x084f4100 },
+	{ _MMIO(0x9888), 0x0a4f0061 },
+	{ _MMIO(0x9888), 0x0c4f6c4c },
+	{ _MMIO(0x9888), 0x0e4f4b00 },
+	{ _MMIO(0x9888), 0x1a4f0000 },
+	{ _MMIO(0x9888), 0x1c4f0000 },
+	{ _MMIO(0x9888), 0x180f5000 },
+	{ _MMIO(0x9888), 0x1a0f8800 },
+	{ _MMIO(0x9888), 0x1c0f08a2 },
+	{ _MMIO(0x9888), 0x182c4000 },
+	{ _MMIO(0x9888), 0x1c2c1451 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c0010 },
+	{ _MMIO(0x9888), 0x01938000 },
+	{ _MMIO(0x9888), 0x0f938000 },
+	{ _MMIO(0x9888), 0x19938a28 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x19900177 },
+	{ _MMIO(0x9888), 0x1b900178 },
+	{ _MMIO(0x9888), 0x1d900125 },
+	{ _MMIO(0x9888), 0x1f900123 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x53901000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0c2e001f },
+	{ _MMIO(0x9888), 0x0a2f0000 },
+	{ _MMIO(0x9888), 0x10186800 },
+	{ _MMIO(0x9888), 0x11810019 },
+	{ _MMIO(0x9888), 0x15810013 },
+	{ _MMIO(0x9888), 0x13820020 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x17840000 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x21860000 },
+	{ _MMIO(0x9888), 0x178703e0 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x022e5400 },
+	{ _MMIO(0x9888), 0x002e0000 },
+	{ _MMIO(0x9888), 0x0e2e0080 },
+	{ _MMIO(0x9888), 0x082f0040 },
+	{ _MMIO(0x9888), 0x002f0000 },
+	{ _MMIO(0x9888), 0x06143000 },
+	{ _MMIO(0x9888), 0x06174000 },
+	{ _MMIO(0x9888), 0x06180012 },
+	{ _MMIO(0x9888), 0x00180000 },
+	{ _MMIO(0x9888), 0x0d804000 },
+	{ _MMIO(0x9888), 0x0f804000 },
+	{ _MMIO(0x9888), 0x05804000 },
+	{ _MMIO(0x9888), 0x09810200 },
+	{ _MMIO(0x9888), 0x0b810030 },
+	{ _MMIO(0x9888), 0x03810003 },
+	{ _MMIO(0x9888), 0x21819140 },
+	{ _MMIO(0x9888), 0x23819050 },
+	{ _MMIO(0x9888), 0x25810018 },
+	{ _MMIO(0x9888), 0x0b820980 },
+	{ _MMIO(0x9888), 0x03820d80 },
+	{ _MMIO(0x9888), 0x11820000 },
+	{ _MMIO(0x9888), 0x0182c000 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x09824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0d830004 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x0f831000 },
+	{ _MMIO(0x9888), 0x01848072 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x07848000 },
+	{ _MMIO(0x9888), 0x09844000 },
+	{ _MMIO(0x9888), 0x0f848000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x09860092 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x01869100 },
+	{ _MMIO(0x9888), 0x0f870065 },
+	{ _MMIO(0x9888), 0x01870000 },
+	{ _MMIO(0x9888), 0x19930800 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x1b952000 },
+	{ _MMIO(0x9888), 0x1d955055 },
+	{ _MMIO(0x9888), 0x1f951455 },
+	{ _MMIO(0x9888), 0x0992a000 },
+	{ _MMIO(0x9888), 0x0f928000 },
+	{ _MMIO(0x9888), 0x1192a800 },
+	{ _MMIO(0x9888), 0x1392028a },
+	{ _MMIO(0x9888), 0x0b92a000 },
+	{ _MMIO(0x9888), 0x0d922000 },
+	{ _MMIO(0x9888), 0x13908000 },
+	{ _MMIO(0x9888), 0x21908000 },
+	{ _MMIO(0x9888), 0x23908000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27908000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900c01 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900863 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900061 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x45900c22 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x19800343 },
+	{ _MMIO(0x9888), 0x39900340 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x41900003 },
+	{ _MMIO(0x9888), 0x03803180 },
+	{ _MMIO(0x9888), 0x058035e2 },
+	{ _MMIO(0x9888), 0x0780006a },
+	{ _MMIO(0x9888), 0x11800000 },
+	{ _MMIO(0x9888), 0x2181a000 },
+	{ _MMIO(0x9888), 0x2381000a },
+	{ _MMIO(0x9888), 0x1d950550 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d92a000 },
+	{ _MMIO(0x9888), 0x0f922000 },
+	{ _MMIO(0x9888), 0x13900170 },
+	{ _MMIO(0x9888), 0x21900171 },
+	{ _MMIO(0x9888), 0x23900172 },
+	{ _MMIO(0x9888), 0x25900173 },
+	{ _MMIO(0x9888), 0x27900174 },
+	{ _MMIO(0x9888), 0x29900175 },
+	{ _MMIO(0x9888), 0x2b900176 },
+	{ _MMIO(0x9888), 0x2d900177 },
+	{ _MMIO(0x9888), 0x2f90017f },
+	{ _MMIO(0x9888), 0x31900125 },
+	{ _MMIO(0x9888), 0x15900123 },
+	{ _MMIO(0x9888), 0x17900121 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43901084 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47901080 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49901084 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b901084 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900004 },
+	{ _MMIO(0x9888), 0x45900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x19800343 },
+	{ _MMIO(0x9888), 0x39900340 },
+	{ _MMIO(0x9888), 0x3f900000 },
+	{ _MMIO(0x9888), 0x41900080 },
+	{ _MMIO(0x9888), 0x03803180 },
+	{ _MMIO(0x9888), 0x058035e2 },
+	{ _MMIO(0x9888), 0x0780006a },
+	{ _MMIO(0x9888), 0x11800000 },
+	{ _MMIO(0x9888), 0x2181a000 },
+	{ _MMIO(0x9888), 0x2381000a },
+	{ _MMIO(0x9888), 0x1d950550 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d92a000 },
+	{ _MMIO(0x9888), 0x0f922000 },
+	{ _MMIO(0x9888), 0x13900180 },
+	{ _MMIO(0x9888), 0x21900181 },
+	{ _MMIO(0x9888), 0x23900182 },
+	{ _MMIO(0x9888), 0x25900183 },
+	{ _MMIO(0x9888), 0x27900184 },
+	{ _MMIO(0x9888), 0x29900185 },
+	{ _MMIO(0x9888), 0x2b900186 },
+	{ _MMIO(0x9888), 0x2d900187 },
+	{ _MMIO(0x9888), 0x2f900170 },
+	{ _MMIO(0x9888), 0x31900125 },
+	{ _MMIO(0x9888), 0x15900123 },
+	{ _MMIO(0x9888), 0x17900121 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43901084 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47901080 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49901084 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b901084 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900004 },
+	{ _MMIO(0x9888), 0x45900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x141c0160 },
+	{ _MMIO(0x9888), 0x161c0015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x002d5000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d5000 },
+	{ _MMIO(0x9888), 0x0a2d5000 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x0c2e5400 },
+	{ _MMIO(0x9888), 0x0e2e5515 },
+	{ _MMIO(0x9888), 0x102e0155 },
+	{ _MMIO(0x9888), 0x044cc000 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4cc000 },
+	{ _MMIO(0x9888), 0x0e4cc000 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x004ea000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084ea000 },
+	{ _MMIO(0x9888), 0x0a4ea000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x0e4f4b41 },
+	{ _MMIO(0x9888), 0x004f4200 },
+	{ _MMIO(0x9888), 0x024f404c },
+	{ _MMIO(0x9888), 0x1c4f0000 },
+	{ _MMIO(0x9888), 0x1a4f0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0a1bc000 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x001c0031 },
+	{ _MMIO(0x9888), 0x061c1900 },
+	{ _MMIO(0x9888), 0x081c1a33 },
+	{ _MMIO(0x9888), 0x0a1c1b35 },
+	{ _MMIO(0x9888), 0x0c1c3337 },
+	{ _MMIO(0x9888), 0x041c31c7 },
+	{ _MMIO(0x9888), 0x180f5000 },
+	{ _MMIO(0x9888), 0x1a0fa8aa },
+	{ _MMIO(0x9888), 0x1c0f0aaa },
+	{ _MMIO(0x9888), 0x182c8000 },
+	{ _MMIO(0x9888), 0x1c2c6aaa },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c2950 },
+	{ _MMIO(0x9888), 0x01938000 },
+	{ _MMIO(0x9888), 0x0f938000 },
+	{ _MMIO(0x9888), 0x1993aaaa },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29904000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900420 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900400 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x45900001 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c03b0 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f900c00 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x002d1000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x082d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x0c2e0400 },
+	{ _MMIO(0x9888), 0x0e2e1500 },
+	{ _MMIO(0x9888), 0x102e0140 },
+	{ _MMIO(0x9888), 0x044c4000 },
+	{ _MMIO(0x9888), 0x0a4c8000 },
+	{ _MMIO(0x9888), 0x0c4cc000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x004e2000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x1a4f4001 },
+	{ _MMIO(0x9888), 0x1c4f5005 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x180f1000 },
+	{ _MMIO(0x9888), 0x1a0fa800 },
+	{ _MMIO(0x9888), 0x1c0f0a00 },
+	{ _MMIO(0x9888), 0x182c4000 },
+	{ _MMIO(0x9888), 0x1c2c4015 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x03931980 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x01938000 },
+	{ _MMIO(0x9888), 0x0f938000 },
+	{ _MMIO(0x9888), 0x1993a00a },
+	{ _MMIO(0x9888), 0x07930000 },
+	{ _MMIO(0x9888), 0x09930000 },
+	{ _MMIO(0x9888), 0x1d900177 },
+	{ _MMIO(0x9888), 0x1f900178 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x53901000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x47900001 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x022d4000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0e2e0055 },
+	{ _MMIO(0x9888), 0x064c8000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x024f6100 },
+	{ _MMIO(0x9888), 0x044f416b },
+	{ _MMIO(0x9888), 0x064f004b },
+	{ _MMIO(0x9888), 0x1a4f0000 },
+	{ _MMIO(0x9888), 0x1a0f02a8 },
+	{ _MMIO(0x9888), 0x1a2c5500 },
+	{ _MMIO(0x9888), 0x0f808000 },
+	{ _MMIO(0x9888), 0x25810020 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x1f951000 },
+	{ _MMIO(0x9888), 0x13920200 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4d900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x12643400 },
+	{ _MMIO(0x9888), 0x12653400 },
+	{ _MMIO(0x9888), 0x106c6800 },
+	{ _MMIO(0x9888), 0x126c001e },
+	{ _MMIO(0x9888), 0x166c0010 },
+	{ _MMIO(0x9888), 0x0c2d5000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e0154 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0055 },
+	{ _MMIO(0x9888), 0x104c8000 },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0c4ea000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c4f5500 },
+	{ _MMIO(0x9888), 0x1a4f1554 },
+	{ _MMIO(0x9888), 0x0a640024 },
+	{ _MMIO(0x9888), 0x10640000 },
+	{ _MMIO(0x9888), 0x04640000 },
+	{ _MMIO(0x9888), 0x0c650024 },
+	{ _MMIO(0x9888), 0x10650000 },
+	{ _MMIO(0x9888), 0x06650000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0900 },
+	{ _MMIO(0x9888), 0x1c0f0aa0 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f02aa },
+	{ _MMIO(0x9888), 0x1c2c5400 },
+	{ _MMIO(0x9888), 0x1e2c0001 },
+	{ _MMIO(0x9888), 0x1a2c5550 },
+	{ _MMIO(0x9888), 0x1993aa00 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900421 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900420 },
+	{ _MMIO(0x9888), 0x45900021 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102d7800 },
+	{ _MMIO(0x9888), 0x122d79e0 },
+	{ _MMIO(0x9888), 0x0c2f0004 },
+	{ _MMIO(0x9888), 0x100e3800 },
+	{ _MMIO(0x9888), 0x180f0005 },
+	{ _MMIO(0x9888), 0x002d0940 },
+	{ _MMIO(0x9888), 0x022d802f },
+	{ _MMIO(0x9888), 0x042d4013 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0e2e0050 },
+	{ _MMIO(0x9888), 0x022f0010 },
+	{ _MMIO(0x9888), 0x002f0000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x040e0480 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x060f0027 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x1a0f0040 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x439014a0 },
+	{ _MMIO(0x9888), 0x459000a4 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x121300a0 },
+	{ _MMIO(0x9888), 0x141600ab },
+	{ _MMIO(0x9888), 0x123300a0 },
+	{ _MMIO(0x9888), 0x143600ab },
+	{ _MMIO(0x9888), 0x125300a0 },
+	{ _MMIO(0x9888), 0x145600ab },
+	{ _MMIO(0x9888), 0x0c2d4000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e01a0 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0065 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x084c4000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x044e2000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c0f0800 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f023f },
+	{ _MMIO(0x9888), 0x1e2c0003 },
+	{ _MMIO(0x9888), 0x1a2cc030 },
+	{ _MMIO(0x9888), 0x04132180 },
+	{ _MMIO(0x9888), 0x02130000 },
+	{ _MMIO(0x9888), 0x0c148000 },
+	{ _MMIO(0x9888), 0x0e142000 },
+	{ _MMIO(0x9888), 0x04148000 },
+	{ _MMIO(0x9888), 0x1e150140 },
+	{ _MMIO(0x9888), 0x1c150040 },
+	{ _MMIO(0x9888), 0x0c163000 },
+	{ _MMIO(0x9888), 0x0e160068 },
+	{ _MMIO(0x9888), 0x10160000 },
+	{ _MMIO(0x9888), 0x18160000 },
+	{ _MMIO(0x9888), 0x0a164000 },
+	{ _MMIO(0x9888), 0x04330043 },
+	{ _MMIO(0x9888), 0x02330000 },
+	{ _MMIO(0x9888), 0x0234a000 },
+	{ _MMIO(0x9888), 0x04342000 },
+	{ _MMIO(0x9888), 0x1c350015 },
+	{ _MMIO(0x9888), 0x02363460 },
+	{ _MMIO(0x9888), 0x10360000 },
+	{ _MMIO(0x9888), 0x04360000 },
+	{ _MMIO(0x9888), 0x06360000 },
+	{ _MMIO(0x9888), 0x08364000 },
+	{ _MMIO(0x9888), 0x06530043 },
+	{ _MMIO(0x9888), 0x02530000 },
+	{ _MMIO(0x9888), 0x0e548000 },
+	{ _MMIO(0x9888), 0x00548000 },
+	{ _MMIO(0x9888), 0x06542000 },
+	{ _MMIO(0x9888), 0x1e550400 },
+	{ _MMIO(0x9888), 0x1a552000 },
+	{ _MMIO(0x9888), 0x1c550100 },
+	{ _MMIO(0x9888), 0x0e563000 },
+	{ _MMIO(0x9888), 0x00563400 },
+	{ _MMIO(0x9888), 0x10560000 },
+	{ _MMIO(0x9888), 0x18560000 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x0c564000 },
+	{ _MMIO(0x9888), 0x1993a800 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b9014a0 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900001 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900820 },
+	{ _MMIO(0x9888), 0x45901022 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x141a0000 },
+	{ _MMIO(0x9888), 0x143a0000 },
+	{ _MMIO(0x9888), 0x145a0000 },
+	{ _MMIO(0x9888), 0x0c2d4000 },
+	{ _MMIO(0x9888), 0x0e2d5000 },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x102e0150 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e006a },
+	{ _MMIO(0x9888), 0x124c8000 },
+	{ _MMIO(0x9888), 0x144c8000 },
+	{ _MMIO(0x9888), 0x164c2000 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064c4000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x0e4ea000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024e2000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x1c0f0bc0 },
+	{ _MMIO(0x9888), 0x180f4000 },
+	{ _MMIO(0x9888), 0x1a0f0302 },
+	{ _MMIO(0x9888), 0x1e2c0003 },
+	{ _MMIO(0x9888), 0x1a2c00f0 },
+	{ _MMIO(0x9888), 0x021a3080 },
+	{ _MMIO(0x9888), 0x041a31e5 },
+	{ _MMIO(0x9888), 0x02148000 },
+	{ _MMIO(0x9888), 0x0414a000 },
+	{ _MMIO(0x9888), 0x1c150054 },
+	{ _MMIO(0x9888), 0x06168000 },
+	{ _MMIO(0x9888), 0x08168000 },
+	{ _MMIO(0x9888), 0x0a168000 },
+	{ _MMIO(0x9888), 0x0c3a3280 },
+	{ _MMIO(0x9888), 0x0e3a0063 },
+	{ _MMIO(0x9888), 0x063a0061 },
+	{ _MMIO(0x9888), 0x023a0000 },
+	{ _MMIO(0x9888), 0x0c348000 },
+	{ _MMIO(0x9888), 0x0e342000 },
+	{ _MMIO(0x9888), 0x06342000 },
+	{ _MMIO(0x9888), 0x1e350140 },
+	{ _MMIO(0x9888), 0x1c350100 },
+	{ _MMIO(0x9888), 0x18360028 },
+	{ _MMIO(0x9888), 0x0c368000 },
+	{ _MMIO(0x9888), 0x0e5a3080 },
+	{ _MMIO(0x9888), 0x005a3280 },
+	{ _MMIO(0x9888), 0x025a0063 },
+	{ _MMIO(0x9888), 0x0e548000 },
+	{ _MMIO(0x9888), 0x00548000 },
+	{ _MMIO(0x9888), 0x02542000 },
+	{ _MMIO(0x9888), 0x1e550400 },
+	{ _MMIO(0x9888), 0x1a552000 },
+	{ _MMIO(0x9888), 0x1c550001 },
+	{ _MMIO(0x9888), 0x18560080 },
+	{ _MMIO(0x9888), 0x02568000 },
+	{ _MMIO(0x9888), 0x04568000 },
+	{ _MMIO(0x9888), 0x1993a800 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x2d904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4d900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x45901084 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900001 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x141a026b },
+	{ _MMIO(0x9888), 0x143a0173 },
+	{ _MMIO(0x9888), 0x145a026b },
+	{ _MMIO(0x9888), 0x002d4000 },
+	{ _MMIO(0x9888), 0x022d5000 },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0c2e5000 },
+	{ _MMIO(0x9888), 0x0e2e0069 },
+	{ _MMIO(0x9888), 0x044c8000 },
+	{ _MMIO(0x9888), 0x064cc000 },
+	{ _MMIO(0x9888), 0x0a4c4000 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x024ea000 },
+	{ _MMIO(0x9888), 0x064e2000 },
+	{ _MMIO(0x9888), 0x180f6000 },
+	{ _MMIO(0x9888), 0x1a0f030a },
+	{ _MMIO(0x9888), 0x1a2c03c0 },
+	{ _MMIO(0x9888), 0x041a37e7 },
+	{ _MMIO(0x9888), 0x021a0000 },
+	{ _MMIO(0x9888), 0x0414a000 },
+	{ _MMIO(0x9888), 0x1c150050 },
+	{ _MMIO(0x9888), 0x08168000 },
+	{ _MMIO(0x9888), 0x0a168000 },
+	{ _MMIO(0x9888), 0x003a3380 },
+	{ _MMIO(0x9888), 0x063a006f },
+	{ _MMIO(0x9888), 0x023a0000 },
+	{ _MMIO(0x9888), 0x00348000 },
+	{ _MMIO(0x9888), 0x06342000 },
+	{ _MMIO(0x9888), 0x1a352000 },
+	{ _MMIO(0x9888), 0x1c350100 },
+	{ _MMIO(0x9888), 0x02368000 },
+	{ _MMIO(0x9888), 0x0c368000 },
+	{ _MMIO(0x9888), 0x025a37e7 },
+	{ _MMIO(0x9888), 0x0254a000 },
+	{ _MMIO(0x9888), 0x1c550005 },
+	{ _MMIO(0x9888), 0x04568000 },
+	{ _MMIO(0x9888), 0x06568000 },
+	{ _MMIO(0x9888), 0x03938000 },
+	{ _MMIO(0x9888), 0x05938000 },
+	{ _MMIO(0x9888), 0x07938000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17904000 },
+	{ _MMIO(0x9888), 0x19904000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900020 },
+	{ _MMIO(0x9888), 0x45901080 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+	{ _MMIO(0xe458), 0x00001000 },
+	{ _MMIO(0xe558), 0x00003002 },
+	{ _MMIO(0xe658), 0x00005004 },
+	{ _MMIO(0xe758), 0x00011010 },
+	{ _MMIO(0xe45c), 0x00050012 },
+	{ _MMIO(0xe55c), 0x00052051 },
+	{ _MMIO(0xe65c), 0x00000008 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x141a001f },
+	{ _MMIO(0x9888), 0x143a001f },
+	{ _MMIO(0x9888), 0x145a001f },
+	{ _MMIO(0x9888), 0x042d5000 },
+	{ _MMIO(0x9888), 0x062d1000 },
+	{ _MMIO(0x9888), 0x0e2e0094 },
+	{ _MMIO(0x9888), 0x084cc000 },
+	{ _MMIO(0x9888), 0x044ea000 },
+	{ _MMIO(0x9888), 0x1a0f00e0 },
+	{ _MMIO(0x9888), 0x1a2c0c00 },
+	{ _MMIO(0x9888), 0x061a0063 },
+	{ _MMIO(0x9888), 0x021a0000 },
+	{ _MMIO(0x9888), 0x06142000 },
+	{ _MMIO(0x9888), 0x1c150100 },
+	{ _MMIO(0x9888), 0x0c168000 },
+	{ _MMIO(0x9888), 0x043a3180 },
+	{ _MMIO(0x9888), 0x023a0000 },
+	{ _MMIO(0x9888), 0x04348000 },
+	{ _MMIO(0x9888), 0x1c350040 },
+	{ _MMIO(0x9888), 0x0a368000 },
+	{ _MMIO(0x9888), 0x045a0063 },
+	{ _MMIO(0x9888), 0x025a0000 },
+	{ _MMIO(0x9888), 0x04542000 },
+	{ _MMIO(0x9888), 0x1c550010 },
+	{ _MMIO(0x9888), 0x08568000 },
+	{ _MMIO(0x9888), 0x09938000 },
+	{ _MMIO(0x9888), 0x0b938000 },
+	{ _MMIO(0x9888), 0x0d938000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1d904000 },
+	{ _MMIO(0x9888), 0x1f904000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900004 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x19800000 },
+	{ _MMIO(0x9888), 0x07800063 },
+	{ _MMIO(0x9888), 0x11800000 },
+	{ _MMIO(0x9888), 0x23810008 },
+	{ _MMIO(0x9888), 0x1d950400 },
+	{ _MMIO(0x9888), 0x0f922000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_glk(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "d72df5c7-5b4a-4274-a43f-00b0fd51fc68",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "814285f6-354d-41d2-ba49-e24e622714a0",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "07d397a6-b3e6-49f6-9433-a4f293d55978",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "1a356946-5428-450b-a2f0-89f8783a302d",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "5299be9d-7a61-4c99-9f81-f87e6c5aaca9",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "bc9bcff2-459a-4cbc-986d-a84b077153f3",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "88ec931f-5b4a-453a-9db6-a61232b6143d",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "530d176d-2a18-4014-adf8-1500c6c60835",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "fdee5a5a-f23c-43d1-aa73-f6257c71671d",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "6617623e-ca73-4791-b2b7-ddedd0846a0c",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "f3b2ea63-e82e-4234-b418-44dd20dd34d0",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "14411d35-cbf6-4f5e-b68b-190faf9a1a83",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "ffa3f263-0478-4724-8c9f-c911c5ec0f1d",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "15274c82-27d2-4819-876a-7cb1a2c59ba4",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "dd3fd789-e783-4204-8cd0-b671bbccb0cf",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_glk(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_glk(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_glk.h b/drivers/gpu/drm/i915/i915_oa_glk.h
new file mode 100644
index 000000000000..5511bb1cecf7
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_glk.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_GLK_H__
+#define __I915_OA_GLK_H__
+
+extern int i915_oa_n_builtin_metric_sets_glk;
+
+extern int i915_oa_select_metric_set_glk(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_glk(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_glk(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.c b/drivers/gpu/drm/i915/i915_oa_hsw.c
index 4ddf756add31..10f169f683b7 100644
--- a/drivers/gpu/drm/i915/i915_oa_hsw.c
+++ b/drivers/gpu/drm/i915/i915_oa_hsw.c
@@ -1,5 +1,7 @@
 /*
- * Autogenerated file, DO NOT EDIT manually!
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
  *
  * Copyright (c) 2015 Intel Corporation
  *
@@ -47,6 +49,9 @@ static const struct i915_oa_reg b_counter_config_render_basic[] = {
 	{ _MMIO(0x2710), 0x00000000 },
 };
 
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+};
+
 static const struct i915_oa_reg mux_config_render_basic[] = {
 	{ _MMIO(0x253a4), 0x01600000 },
 	{ _MMIO(0x25440), 0x00100000 },
@@ -109,12 +114,21 @@ static const struct i915_oa_reg mux_config_render_basic[] = {
 	{ _MMIO(0x25428), 0x00042049 },
 };
 
-static const struct i915_oa_reg *
+static int
 get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    int *len)
+			    const struct i915_oa_reg **regs,
+			    int *lens)
 {
-	*len = ARRAY_SIZE(mux_config_render_basic);
-	return mux_config_render_basic;
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
 }
 
 static const struct i915_oa_reg b_counter_config_compute_basic[] = {
@@ -137,6 +151,9 @@ static const struct i915_oa_reg b_counter_config_compute_basic[] = {
 	{ _MMIO(0x236c), 0x00000000 },
 };
 
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+};
+
 static const struct i915_oa_reg mux_config_compute_basic[] = {
 	{ _MMIO(0x253a4), 0x00000000 },
 	{ _MMIO(0x2681c), 0x01f00800 },
@@ -172,12 +189,21 @@ static const struct i915_oa_reg mux_config_compute_basic[] = {
 	{ _MMIO(0x25428), 0x00000c03 },
 };
 
-static const struct i915_oa_reg *
+static int
 get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     int *len)
+			     const struct i915_oa_reg **regs,
+			     int *lens)
 {
-	*len = ARRAY_SIZE(mux_config_compute_basic);
-	return mux_config_compute_basic;
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
 }
 
 static const struct i915_oa_reg b_counter_config_compute_extended[] = {
@@ -203,6 +229,9 @@ static const struct i915_oa_reg b_counter_config_compute_extended[] = {
 	{ _MMIO(0x27ac), 0x0000fffe },
 };
 
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+};
+
 static const struct i915_oa_reg mux_config_compute_extended[] = {
 	{ _MMIO(0x2681c), 0x3eb00800 },
 	{ _MMIO(0x26820), 0x00900000 },
@@ -221,12 +250,21 @@ static const struct i915_oa_reg mux_config_compute_extended[] = {
 	{ _MMIO(0x25428), 0x00000000 },
 };
 
-static const struct i915_oa_reg *
+static int
 get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				int *len)
+				const struct i915_oa_reg **regs,
+				int *lens)
 {
-	*len = ARRAY_SIZE(mux_config_compute_extended);
-	return mux_config_compute_extended;
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
 }
 
 static const struct i915_oa_reg b_counter_config_memory_reads[] = {
@@ -260,6 +298,9 @@ static const struct i915_oa_reg b_counter_config_memory_reads[] = {
 	{ _MMIO(0x27ac), 0x0000fc00 },
 };
 
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+};
+
 static const struct i915_oa_reg mux_config_memory_reads[] = {
 	{ _MMIO(0x253a4), 0x34300000 },
 	{ _MMIO(0x25440), 0x2d800000 },
@@ -281,12 +322,21 @@ static const struct i915_oa_reg mux_config_memory_reads[] = {
 	{ _MMIO(0x25428), 0x00000000 },
 };
 
-static const struct i915_oa_reg *
+static int
 get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    int *len)
+			    const struct i915_oa_reg **regs,
+			    int *lens)
 {
-	*len = ARRAY_SIZE(mux_config_memory_reads);
-	return mux_config_memory_reads;
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
 }
 
 static const struct i915_oa_reg b_counter_config_memory_writes[] = {
@@ -320,6 +370,9 @@ static const struct i915_oa_reg b_counter_config_memory_writes[] = {
 	{ _MMIO(0x27ac), 0x0000fc00 },
 };
 
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+};
+
 static const struct i915_oa_reg mux_config_memory_writes[] = {
 	{ _MMIO(0x253a4), 0x34300000 },
 	{ _MMIO(0x25440), 0x01500000 },
@@ -341,12 +394,21 @@ static const struct i915_oa_reg mux_config_memory_writes[] = {
 	{ _MMIO(0x25428), 0x00000000 },
 };
 
-static const struct i915_oa_reg *
+static int
 get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     int *len)
+			     const struct i915_oa_reg **regs,
+			     int *lens)
 {
-	*len = ARRAY_SIZE(mux_config_memory_writes);
-	return mux_config_memory_writes;
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
 }
 
 static const struct i915_oa_reg b_counter_config_sampler_balance[] = {
@@ -358,6 +420,9 @@ static const struct i915_oa_reg b_counter_config_sampler_balance[] = {
 	{ _MMIO(0x2724), 0x00800000 },
 };
 
+static const struct i915_oa_reg flex_eu_config_sampler_balance[] = {
+};
+
 static const struct i915_oa_reg mux_config_sampler_balance[] = {
 	{ _MMIO(0x2eb9c), 0x01906400 },
 	{ _MMIO(0x2fb9c), 0x01906400 },
@@ -401,31 +466,40 @@ static const struct i915_oa_reg mux_config_sampler_balance[] = {
 	{ _MMIO(0x25428), 0x0004a54a },
 };
 
-static const struct i915_oa_reg *
+static int
 get_sampler_balance_mux_config(struct drm_i915_private *dev_priv,
-			       int *len)
+			       const struct i915_oa_reg **regs,
+			       int *lens)
 {
-	*len = ARRAY_SIZE(mux_config_sampler_balance);
-	return mux_config_sampler_balance;
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler_balance;
+	lens[n] = ARRAY_SIZE(mux_config_sampler_balance);
+	n++;
+
+	return n;
 }
 
 int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 {
-	dev_priv->perf.oa.mux_regs = NULL;
-	dev_priv->perf.oa.mux_regs_len = 0;
+	dev_priv->perf.oa.n_mux_configs = 0;
 	dev_priv->perf.oa.b_counter_regs = NULL;
 	dev_priv->perf.oa.b_counter_regs_len = 0;
 
 	switch (dev_priv->perf.oa.metrics_set) {
 	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.mux_regs =
+		dev_priv->perf.oa.n_mux_configs =
 			get_render_basic_mux_config(dev_priv,
-						    &dev_priv->perf.oa.mux_regs_len);
-		if (!dev_priv->perf.oa.mux_regs) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set");
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
 
 			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised so userspace and
+			 * and so it wouldn't have been advertised to userspace and
 			 * so shouldn't have been requested
 			 */
 			return -EINVAL;
@@ -436,16 +510,22 @@ int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.b_counter_regs_len =
 			ARRAY_SIZE(b_counter_config_render_basic);
 
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
 		return 0;
 	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.mux_regs =
+		dev_priv->perf.oa.n_mux_configs =
 			get_compute_basic_mux_config(dev_priv,
-						     &dev_priv->perf.oa.mux_regs_len);
-		if (!dev_priv->perf.oa.mux_regs) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set");
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
 
 			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised so userspace and
+			 * and so it wouldn't have been advertised to userspace and
 			 * so shouldn't have been requested
 			 */
 			return -EINVAL;
@@ -456,16 +536,22 @@ int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.b_counter_regs_len =
 			ARRAY_SIZE(b_counter_config_compute_basic);
 
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
 		return 0;
 	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.mux_regs =
+		dev_priv->perf.oa.n_mux_configs =
 			get_compute_extended_mux_config(dev_priv,
-							&dev_priv->perf.oa.mux_regs_len);
-		if (!dev_priv->perf.oa.mux_regs) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set");
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
 
 			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised so userspace and
+			 * and so it wouldn't have been advertised to userspace and
 			 * so shouldn't have been requested
 			 */
 			return -EINVAL;
@@ -476,16 +562,22 @@ int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.b_counter_regs_len =
 			ARRAY_SIZE(b_counter_config_compute_extended);
 
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
 		return 0;
 	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.mux_regs =
+		dev_priv->perf.oa.n_mux_configs =
 			get_memory_reads_mux_config(dev_priv,
-						    &dev_priv->perf.oa.mux_regs_len);
-		if (!dev_priv->perf.oa.mux_regs) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set");
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
 
 			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised so userspace and
+			 * and so it wouldn't have been advertised to userspace and
 			 * so shouldn't have been requested
 			 */
 			return -EINVAL;
@@ -496,16 +588,22 @@ int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.b_counter_regs_len =
 			ARRAY_SIZE(b_counter_config_memory_reads);
 
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
 		return 0;
 	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.mux_regs =
+		dev_priv->perf.oa.n_mux_configs =
 			get_memory_writes_mux_config(dev_priv,
-						     &dev_priv->perf.oa.mux_regs_len);
-		if (!dev_priv->perf.oa.mux_regs) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set");
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
 
 			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised so userspace and
+			 * and so it wouldn't have been advertised to userspace and
 			 * so shouldn't have been requested
 			 */
 			return -EINVAL;
@@ -516,16 +614,22 @@ int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.b_counter_regs_len =
 			ARRAY_SIZE(b_counter_config_memory_writes);
 
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
 		return 0;
 	case METRIC_SET_ID_SAMPLER_BALANCE:
-		dev_priv->perf.oa.mux_regs =
+		dev_priv->perf.oa.n_mux_configs =
 			get_sampler_balance_mux_config(dev_priv,
-						       &dev_priv->perf.oa.mux_regs_len);
-		if (!dev_priv->perf.oa.mux_regs) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_BALANCE\" metric set");
+						       dev_priv->perf.oa.mux_regs,
+						       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_BALANCE\" metric set\n");
 
 			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised so userspace and
+			 * and so it wouldn't have been advertised to userspace and
 			 * so shouldn't have been requested
 			 */
 			return -EINVAL;
@@ -536,6 +640,11 @@ int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.b_counter_regs_len =
 			ARRAY_SIZE(b_counter_config_sampler_balance);
 
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler_balance;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler_balance);
+
 		return 0;
 	default:
 		return -ENODEV;
@@ -677,35 +786,36 @@ static struct attribute_group group_sampler_balance = {
 int
 i915_perf_register_sysfs_hsw(struct drm_i915_private *dev_priv)
 {
-	int mux_len;
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
 	int ret = 0;
 
-	if (get_render_basic_mux_config(dev_priv, &mux_len)) {
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
 		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
 		if (ret)
 			goto error_render_basic;
 	}
-	if (get_compute_basic_mux_config(dev_priv, &mux_len)) {
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
 		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
 		if (ret)
 			goto error_compute_basic;
 	}
-	if (get_compute_extended_mux_config(dev_priv, &mux_len)) {
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
 		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
 		if (ret)
 			goto error_compute_extended;
 	}
-	if (get_memory_reads_mux_config(dev_priv, &mux_len)) {
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
 		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
 		if (ret)
 			goto error_memory_reads;
 	}
-	if (get_memory_writes_mux_config(dev_priv, &mux_len)) {
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
 		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
 		if (ret)
 			goto error_memory_writes;
 	}
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len)) {
+	if (get_sampler_balance_mux_config(dev_priv, mux_regs, mux_lens)) {
 		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_balance);
 		if (ret)
 			goto error_sampler_balance;
@@ -714,19 +824,19 @@ i915_perf_register_sysfs_hsw(struct drm_i915_private *dev_priv)
 	return 0;
 
 error_sampler_balance:
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len))
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
 error_memory_writes:
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len))
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
 error_memory_reads:
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len))
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
 error_compute_extended:
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len))
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
 error_compute_basic:
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len))
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
 error_render_basic:
 	return ret;
@@ -735,18 +845,19 @@ error_render_basic:
 void
 i915_perf_unregister_sysfs_hsw(struct drm_i915_private *dev_priv)
 {
-	int mux_len;
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
 
-	if (get_render_basic_mux_config(dev_priv, &mux_len))
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, &mux_len))
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_compute_extended_mux_config(dev_priv, &mux_len))
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_memory_reads_mux_config(dev_priv, &mux_len))
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, &mux_len))
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_sampler_balance_mux_config(dev_priv, &mux_len))
+	if (get_sampler_balance_mux_config(dev_priv, mux_regs, mux_lens))
 		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_balance);
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.h b/drivers/gpu/drm/i915/i915_oa_hsw.h
index 429a229b5158..6fe7e0690ef3 100644
--- a/drivers/gpu/drm/i915/i915_oa_hsw.h
+++ b/drivers/gpu/drm/i915/i915_oa_hsw.h
@@ -1,5 +1,7 @@
 /*
- * Autogenerated file, DO NOT EDIT manually!
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
  *
  * Copyright (c) 2015 Intel Corporation
  *
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.c b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
new file mode 100644
index 000000000000..87dbd0a0b076
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
@@ -0,0 +1,2991 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_kblgt2.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_VME_PIPE,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_kblgt2 = 18;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9888), 0x166c01e0 },
+	{ _MMIO(0x9888), 0x12170280 },
+	{ _MMIO(0x9888), 0x12370280 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x1a4e0080 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x0a1b4000 },
+	{ _MMIO(0x9888), 0x1c1c0001 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x042f1000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c8400 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0d2000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f6600 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x162c2200 },
+	{ _MMIO(0x9888), 0x062d8000 },
+	{ _MMIO(0x9888), 0x082d8000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x08133000 },
+	{ _MMIO(0x9888), 0x00170020 },
+	{ _MMIO(0x9888), 0x08170021 },
+	{ _MMIO(0x9888), 0x10170000 },
+	{ _MMIO(0x9888), 0x0633c000 },
+	{ _MMIO(0x9888), 0x0833c000 },
+	{ _MMIO(0x9888), 0x06370800 },
+	{ _MMIO(0x9888), 0x08370840 },
+	{ _MMIO(0x9888), 0x10370000 },
+	{ _MMIO(0x9888), 0x0d933031 },
+	{ _MMIO(0x9888), 0x0f933e3f },
+	{ _MMIO(0x9888), 0x01933d00 },
+	{ _MMIO(0x9888), 0x0393073c },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1d930000 },
+	{ _MMIO(0x9888), 0x19930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190001f },
+	{ _MMIO(0x9888), 0x51904400 },
+	{ _MMIO(0x9888), 0x41900020 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c21 },
+	{ _MMIO(0x9888), 0x47900061 },
+	{ _MMIO(0x9888), 0x57904440 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900004 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53904444 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x1a4e0820 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x064f0900 },
+	{ _MMIO(0x9888), 0x084f0032 },
+	{ _MMIO(0x9888), 0x0a4f1891 },
+	{ _MMIO(0x9888), 0x0c4f0e00 },
+	{ _MMIO(0x9888), 0x0e4f003c },
+	{ _MMIO(0x9888), 0x004f0d80 },
+	{ _MMIO(0x9888), 0x024f003b },
+	{ _MMIO(0x9888), 0x006c0002 },
+	{ _MMIO(0x9888), 0x086c0100 },
+	{ _MMIO(0x9888), 0x0c6c000c },
+	{ _MMIO(0x9888), 0x0e6c0b00 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x081b8000 },
+	{ _MMIO(0x9888), 0x0c1b4000 },
+	{ _MMIO(0x9888), 0x0e1b8000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1c8000 },
+	{ _MMIO(0x9888), 0x1c1c0024 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5bc000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x1a5c6000 },
+	{ _MMIO(0x9888), 0x1c5c001b },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2000 },
+	{ _MMIO(0x9888), 0x0c4c0208 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020d2000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2cc000 },
+	{ _MMIO(0x9888), 0x162cfb00 },
+	{ _MMIO(0x9888), 0x182c00be },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x19900157 },
+	{ _MMIO(0x9888), 0x1b900158 },
+	{ _MMIO(0x9888), 0x1d900105 },
+	{ _MMIO(0x9888), 0x1f900103 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x11900fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900821 },
+	{ _MMIO(0x9888), 0x47900802 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900802 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900002 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900422 },
+	{ _MMIO(0x9888), 0x53904444 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0c0e001f },
+	{ _MMIO(0x9888), 0x0a0f0000 },
+	{ _MMIO(0x9888), 0x10116800 },
+	{ _MMIO(0x9888), 0x178a03e0 },
+	{ _MMIO(0x9888), 0x11824c00 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x13840020 },
+	{ _MMIO(0x9888), 0x11850019 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x01870c40 },
+	{ _MMIO(0x9888), 0x17880000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x040d4000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020e5400 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x080f0040 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x0e0f0040 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06110012 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x01898000 },
+	{ _MMIO(0x9888), 0x0d890100 },
+	{ _MMIO(0x9888), 0x03898000 },
+	{ _MMIO(0x9888), 0x09808000 },
+	{ _MMIO(0x9888), 0x0b808000 },
+	{ _MMIO(0x9888), 0x0380c000 },
+	{ _MMIO(0x9888), 0x0f8a0075 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x118a8000 },
+	{ _MMIO(0x9888), 0x1b8a4000 },
+	{ _MMIO(0x9888), 0x138a8000 },
+	{ _MMIO(0x9888), 0x1d81a000 },
+	{ _MMIO(0x9888), 0x15818000 },
+	{ _MMIO(0x9888), 0x17818000 },
+	{ _MMIO(0x9888), 0x0b820030 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x0d824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x05824000 },
+	{ _MMIO(0x9888), 0x0d830003 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x03838000 },
+	{ _MMIO(0x9888), 0x07838000 },
+	{ _MMIO(0x9888), 0x0b840980 },
+	{ _MMIO(0x9888), 0x03844d80 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x09850080 },
+	{ _MMIO(0x9888), 0x03850003 },
+	{ _MMIO(0x9888), 0x01850000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x09870032 },
+	{ _MMIO(0x9888), 0x01888052 },
+	{ _MMIO(0x9888), 0x11880000 },
+	{ _MMIO(0x9888), 0x09884000 },
+	{ _MMIO(0x9888), 0x1b931001 },
+	{ _MMIO(0x9888), 0x1d930001 },
+	{ _MMIO(0x9888), 0x19934000 },
+	{ _MMIO(0x9888), 0x1b958000 },
+	{ _MMIO(0x9888), 0x1d950094 },
+	{ _MMIO(0x9888), 0x19958000 },
+	{ _MMIO(0x9888), 0x09e58000 },
+	{ _MMIO(0x9888), 0x0be58000 },
+	{ _MMIO(0x9888), 0x03e5c000 },
+	{ _MMIO(0x9888), 0x0592c000 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d924000 },
+	{ _MMIO(0x9888), 0x0f924000 },
+	{ _MMIO(0x9888), 0x11928000 },
+	{ _MMIO(0x9888), 0x1392c000 },
+	{ _MMIO(0x9888), 0x09924000 },
+	{ _MMIO(0x9888), 0x01985000 },
+	{ _MMIO(0x9888), 0x07988000 },
+	{ _MMIO(0x9888), 0x09981000 },
+	{ _MMIO(0x9888), 0x0b982000 },
+	{ _MMIO(0x9888), 0x0d982000 },
+	{ _MMIO(0x9888), 0x0f989000 },
+	{ _MMIO(0x9888), 0x05982000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1190c080 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900440 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900c21 },
+	{ _MMIO(0x9888), 0x57900400 },
+	{ _MMIO(0x9888), 0x49900042 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900024 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900841 },
+	{ _MMIO(0x9888), 0x53900400 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900064 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900150 },
+	{ _MMIO(0x9888), 0x21900151 },
+	{ _MMIO(0x9888), 0x23900152 },
+	{ _MMIO(0x9888), 0x25900153 },
+	{ _MMIO(0x9888), 0x27900154 },
+	{ _MMIO(0x9888), 0x29900155 },
+	{ _MMIO(0x9888), 0x2b900156 },
+	{ _MMIO(0x9888), 0x2d900157 },
+	{ _MMIO(0x9888), 0x2f90015f },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900160 },
+	{ _MMIO(0x9888), 0x21900161 },
+	{ _MMIO(0x9888), 0x23900162 },
+	{ _MMIO(0x9888), 0x25900163 },
+	{ _MMIO(0x9888), 0x27900164 },
+	{ _MMIO(0x9888), 0x29900165 },
+	{ _MMIO(0x9888), 0x2b900166 },
+	{ _MMIO(0x9888), 0x2d900167 },
+	{ _MMIO(0x9888), 0x2f900150 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended[] = {
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x141c8160 },
+	{ _MMIO(0x9888), 0x161c8015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4eaaa0 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0e6c0b01 },
+	{ _MMIO(0x9888), 0x006c0200 },
+	{ _MMIO(0x9888), 0x026c000c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x001c0041 },
+	{ _MMIO(0x9888), 0x061c4200 },
+	{ _MMIO(0x9888), 0x081c4443 },
+	{ _MMIO(0x9888), 0x0a1c4645 },
+	{ _MMIO(0x9888), 0x0c1c7647 },
+	{ _MMIO(0x9888), 0x041c7357 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x101c0000 },
+	{ _MMIO(0x9888), 0x1a1c0000 },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4caa2a },
+	{ _MMIO(0x9888), 0x0c4c02aa },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5515 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x11907fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900802 },
+	{ _MMIO(0x9888), 0x47900842 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900842 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900800 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c0760 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4e8020 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1ce000 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2a00 },
+	{ _MMIO(0x9888), 0x0c4c0280 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f1500 },
+	{ _MMIO(0x9888), 0x100f0140 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x182c00a0 },
+	{ _MMIO(0x9888), 0x03933300 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190030f },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900021 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x53904444 },
+	{ _MMIO(0x9888), 0x43900000 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x106c0232 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x004f1880 },
+	{ _MMIO(0x9888), 0x024f08bb },
+	{ _MMIO(0x9888), 0x044f001b },
+	{ _MMIO(0x9888), 0x046c0100 },
+	{ _MMIO(0x9888), 0x066c000b },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x041b8000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025bc000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x165c8000 },
+	{ _MMIO(0x9888), 0x185c8000 },
+	{ _MMIO(0x9888), 0x0a4c00a0 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x062cc000 },
+	{ _MMIO(0x9888), 0x082cc000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x07830000 },
+	{ _MMIO(0x9888), 0x1d950080 },
+	{ _MMIO(0x9888), 0x13928000 },
+	{ _MMIO(0x9888), 0x0f988000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900040 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x126c7b40 },
+	{ _MMIO(0x9888), 0x166c0020 },
+	{ _MMIO(0x9888), 0x0a603444 },
+	{ _MMIO(0x9888), 0x0a613400 },
+	{ _MMIO(0x9888), 0x1a4ea800 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0800 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x1c1c003c },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x10600000 },
+	{ _MMIO(0x9888), 0x04600000 },
+	{ _MMIO(0x9888), 0x0c610044 },
+	{ _MMIO(0x9888), 0x10610000 },
+	{ _MMIO(0x9888), 0x06610000 },
+	{ _MMIO(0x9888), 0x0c4c02a8 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0154 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190ffc0 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900021 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900400 },
+	{ _MMIO(0x9888), 0x43900421 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x126c02e0 },
+	{ _MMIO(0x9888), 0x146c0001 },
+	{ _MMIO(0x9888), 0x0a623400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x026c3324 },
+	{ _MMIO(0x9888), 0x046c3422 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x06614000 },
+	{ _MMIO(0x9888), 0x0c620044 },
+	{ _MMIO(0x9888), 0x10620000 },
+	{ _MMIO(0x9888), 0x06620000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x126c4e80 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x0a633400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x026c3321 },
+	{ _MMIO(0x9888), 0x046c342f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c2000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x06604000 },
+	{ _MMIO(0x9888), 0x0c630044 },
+	{ _MMIO(0x9888), 0x10630000 },
+	{ _MMIO(0x9888), 0x06630000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c00aa },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102f3800 },
+	{ _MMIO(0x9888), 0x144d0500 },
+	{ _MMIO(0x9888), 0x120d03c0 },
+	{ _MMIO(0x9888), 0x140d03cf },
+	{ _MMIO(0x9888), 0x0c0f0004 },
+	{ _MMIO(0x9888), 0x0c4e4000 },
+	{ _MMIO(0x9888), 0x042f0480 },
+	{ _MMIO(0x9888), 0x082f0000 },
+	{ _MMIO(0x9888), 0x022f0000 },
+	{ _MMIO(0x9888), 0x0a4c0090 },
+	{ _MMIO(0x9888), 0x064d0027 },
+	{ _MMIO(0x9888), 0x004d0000 },
+	{ _MMIO(0x9888), 0x000d0d40 },
+	{ _MMIO(0x9888), 0x020d803f },
+	{ _MMIO(0x9888), 0x040d8023 },
+	{ _MMIO(0x9888), 0x100d0000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020f0010 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x0e0f0050 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41901400 },
+	{ _MMIO(0x9888), 0x43901485 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x14152c00 },
+	{ _MMIO(0x9888), 0x16150005 },
+	{ _MMIO(0x9888), 0x121600a0 },
+	{ _MMIO(0x9888), 0x14352c00 },
+	{ _MMIO(0x9888), 0x16350005 },
+	{ _MMIO(0x9888), 0x123600a0 },
+	{ _MMIO(0x9888), 0x14552c00 },
+	{ _MMIO(0x9888), 0x16550005 },
+	{ _MMIO(0x9888), 0x125600a0 },
+	{ _MMIO(0x9888), 0x062f6000 },
+	{ _MMIO(0x9888), 0x022f2000 },
+	{ _MMIO(0x9888), 0x0c4c0050 },
+	{ _MMIO(0x9888), 0x0a4c0010 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0350 },
+	{ _MMIO(0x9888), 0x0c0fb000 },
+	{ _MMIO(0x9888), 0x0e0f00da },
+	{ _MMIO(0x9888), 0x182c0028 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x022dc000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x0c138000 },
+	{ _MMIO(0x9888), 0x0e132000 },
+	{ _MMIO(0x9888), 0x0413c000 },
+	{ _MMIO(0x9888), 0x1c140018 },
+	{ _MMIO(0x9888), 0x0c157000 },
+	{ _MMIO(0x9888), 0x0e150078 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04162180 },
+	{ _MMIO(0x9888), 0x02160000 },
+	{ _MMIO(0x9888), 0x04174000 },
+	{ _MMIO(0x9888), 0x0233a000 },
+	{ _MMIO(0x9888), 0x04333000 },
+	{ _MMIO(0x9888), 0x14348000 },
+	{ _MMIO(0x9888), 0x16348000 },
+	{ _MMIO(0x9888), 0x02357870 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04360043 },
+	{ _MMIO(0x9888), 0x02360000 },
+	{ _MMIO(0x9888), 0x04371000 },
+	{ _MMIO(0x9888), 0x0e538000 },
+	{ _MMIO(0x9888), 0x00538000 },
+	{ _MMIO(0x9888), 0x06533000 },
+	{ _MMIO(0x9888), 0x1c540020 },
+	{ _MMIO(0x9888), 0x12548000 },
+	{ _MMIO(0x9888), 0x0e557000 },
+	{ _MMIO(0x9888), 0x00557800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06560043 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x06571000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900060 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900060 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x12120000 },
+	{ _MMIO(0x9888), 0x12320000 },
+	{ _MMIO(0x9888), 0x12520000 },
+	{ _MMIO(0x9888), 0x002f8000 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0015 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f03a0 },
+	{ _MMIO(0x9888), 0x0c0ff000 },
+	{ _MMIO(0x9888), 0x0e0f0095 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x02108000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x02118000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x02121880 },
+	{ _MMIO(0x9888), 0x041219b5 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x02134000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x0c308000 },
+	{ _MMIO(0x9888), 0x0e304000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x0c318000 },
+	{ _MMIO(0x9888), 0x0e314000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x0c321a80 },
+	{ _MMIO(0x9888), 0x0e320033 },
+	{ _MMIO(0x9888), 0x06320031 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x0c334000 },
+	{ _MMIO(0x9888), 0x0e331000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0e508000 },
+	{ _MMIO(0x9888), 0x00508000 },
+	{ _MMIO(0x9888), 0x02504000 },
+	{ _MMIO(0x9888), 0x0e518000 },
+	{ _MMIO(0x9888), 0x00518000 },
+	{ _MMIO(0x9888), 0x02514000 },
+	{ _MMIO(0x9888), 0x0e521880 },
+	{ _MMIO(0x9888), 0x00521a80 },
+	{ _MMIO(0x9888), 0x02520033 },
+	{ _MMIO(0x9888), 0x0e534000 },
+	{ _MMIO(0x9888), 0x00534000 },
+	{ _MMIO(0x9888), 0x02531000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900062 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x12124d60 },
+	{ _MMIO(0x9888), 0x12322e60 },
+	{ _MMIO(0x9888), 0x12524d60 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0014 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0fe000 },
+	{ _MMIO(0x9888), 0x0e0f0097 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x002d8000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x04121fb7 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x00308000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x00318000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x00321b80 },
+	{ _MMIO(0x9888), 0x0632003f },
+	{ _MMIO(0x9888), 0x00334000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0250c000 },
+	{ _MMIO(0x9888), 0x0251c000 },
+	{ _MMIO(0x9888), 0x02521fb7 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x02535000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900063 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+	{ _MMIO(0xe458), 0x00001000 },
+	{ _MMIO(0xe558), 0x00003002 },
+	{ _MMIO(0xe658), 0x00005004 },
+	{ _MMIO(0xe758), 0x00011010 },
+	{ _MMIO(0xe45c), 0x00050012 },
+	{ _MMIO(0xe55c), 0x00052051 },
+	{ _MMIO(0xe65c), 0x00000008 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x121203e0 },
+	{ _MMIO(0x9888), 0x123203e0 },
+	{ _MMIO(0x9888), 0x125203e0 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0e0f006c },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x042d8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06114000 },
+	{ _MMIO(0x9888), 0x06120033 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x04308000 },
+	{ _MMIO(0x9888), 0x04318000 },
+	{ _MMIO(0x9888), 0x04321980 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x04334000 },
+	{ _MMIO(0x9888), 0x04504000 },
+	{ _MMIO(0x9888), 0x04514000 },
+	{ _MMIO(0x9888), 0x04520033 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x04531000 },
+	{ _MMIO(0x9888), 0x1190e000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900c00 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00100030 },
+	{ _MMIO(0x2774), 0x0000fff9 },
+	{ _MMIO(0x2778), 0x00000002 },
+	{ _MMIO(0x277c), 0x0000fffc },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000fff3 },
+	{ _MMIO(0x2788), 0x00100180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000ff3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00008003 },
+};
+
+static const struct i915_oa_reg mux_config_vme_pipe[] = {
+	{ _MMIO(0x9888), 0x141a5800 },
+	{ _MMIO(0x9888), 0x161a00c0 },
+	{ _MMIO(0x9888), 0x12180240 },
+	{ _MMIO(0x9888), 0x14180002 },
+	{ _MMIO(0x9888), 0x143a5800 },
+	{ _MMIO(0x9888), 0x163a00c0 },
+	{ _MMIO(0x9888), 0x12380240 },
+	{ _MMIO(0x9888), 0x14380002 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x022f8000 },
+	{ _MMIO(0x9888), 0x042f3000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c1500 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f9500 },
+	{ _MMIO(0x9888), 0x100f002a },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x0a2dc000 },
+	{ _MMIO(0x9888), 0x0c2dc000 },
+	{ _MMIO(0x9888), 0x04193000 },
+	{ _MMIO(0x9888), 0x081a28c1 },
+	{ _MMIO(0x9888), 0x001a0000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x0613c000 },
+	{ _MMIO(0x9888), 0x0813f000 },
+	{ _MMIO(0x9888), 0x00172000 },
+	{ _MMIO(0x9888), 0x06178000 },
+	{ _MMIO(0x9888), 0x0817a000 },
+	{ _MMIO(0x9888), 0x00180037 },
+	{ _MMIO(0x9888), 0x06180940 },
+	{ _MMIO(0x9888), 0x08180000 },
+	{ _MMIO(0x9888), 0x02180000 },
+	{ _MMIO(0x9888), 0x04183000 },
+	{ _MMIO(0x9888), 0x06393000 },
+	{ _MMIO(0x9888), 0x0c3a28c1 },
+	{ _MMIO(0x9888), 0x003a0000 },
+	{ _MMIO(0x9888), 0x0a33f000 },
+	{ _MMIO(0x9888), 0x0c33f000 },
+	{ _MMIO(0x9888), 0x0a37a000 },
+	{ _MMIO(0x9888), 0x0c37a000 },
+	{ _MMIO(0x9888), 0x0a380977 },
+	{ _MMIO(0x9888), 0x08380000 },
+	{ _MMIO(0x9888), 0x04380000 },
+	{ _MMIO(0x9888), 0x06383000 },
+	{ _MMIO(0x9888), 0x119000ff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900800 },
+	{ _MMIO(0x9888), 0x47901000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900844 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
+			const struct i915_oa_reg **regs,
+			int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_vme_pipe;
+	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x11810000 },
+	{ _MMIO(0x9888), 0x07810013 },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930040 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_kblgt2(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_VME_PIPE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_vme_pipe_mux_config(dev_priv,
+						dev_priv->perf.oa.mux_regs,
+						dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_vme_pipe;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_vme_pipe);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_vme_pipe;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_vme_pipe);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "f8d677e9-ff6f-4df1-9310-0334c6efacce",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "e17fc42a-e614-41b6-90c4-1074841a6c77",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "d7a17a3a-ca71-40d2-a919-ace80d50633f",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "57b59202-172b-477a-87de-33f85572c589",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "3addf8ef-8e9b-40f5-a448-3dbb5d5128b0",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "4af0400a-81c3-47db-a6b6-deddbd75680e",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "0e22f995-79ca-4f67-83ab-e9d9772488d8",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "bc2a00f7-cb8a-4ff2-8ad0-e241dad16937",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "d2bbe790-f058-42d9-81c6-cdedcf655bc2",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "2f8e32e4-5956-46e2-af31-c8ea95887332",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "ca046aad-b5fb-4101-adce-6473ee6e5b14",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "605f388f-24bb-455c-88e3-8d57ae0d7e9f",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "31dd157c-bf4e-4bab-bf2b-f5c8174af1af",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "105db928-5542-466b-9128-e1f3c91426cb",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "03db94d2-b37f-4c58-a791-0d2067b013bb",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "aa7a3fb9-22fb-43ff-a32d-0ab6c13bbd16",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
+}
+
+static struct device_attribute dev_attr_vme_pipe_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_vme_pipe_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_vme_pipe[] = {
+	&dev_attr_vme_pipe_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_vme_pipe = {
+	.name = "398a4268-ef6f-4ffc-b55f-3c7b5363ce61",
+	.attrs =  attrs_vme_pipe,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "baa3c7e4-52b6-4b85-801e-465a94b746dd",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_kblgt2(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+		if (ret)
+			goto error_vme_pipe;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+error_vme_pipe:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_kblgt2(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.h b/drivers/gpu/drm/i915/i915_oa_kblgt2.h
new file mode 100644
index 000000000000..7e61bfc4f9f5
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_KBLGT2_H__
+#define __I915_OA_KBLGT2_H__
+
+extern int i915_oa_n_builtin_metric_sets_kblgt2;
+
+extern int i915_oa_select_metric_set_kblgt2(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_kblgt2(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_kblgt2(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.c b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
new file mode 100644
index 000000000000..6ed092566a32
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
@@ -0,0 +1,3040 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_kblgt3.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_VME_PIPE,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_kblgt3 = 18;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9888), 0x166c01e0 },
+	{ _MMIO(0x9888), 0x12170280 },
+	{ _MMIO(0x9888), 0x12370280 },
+	{ _MMIO(0x9888), 0x16ec01e0 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x1a4e0380 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x0a1b4000 },
+	{ _MMIO(0x9888), 0x1c1c0001 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x042f1000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c8400 },
+	{ _MMIO(0x9888), 0x0c4c0002 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f6600 },
+	{ _MMIO(0x9888), 0x100f0001 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x162ca200 },
+	{ _MMIO(0x9888), 0x062d8000 },
+	{ _MMIO(0x9888), 0x082d8000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x08133000 },
+	{ _MMIO(0x9888), 0x00170020 },
+	{ _MMIO(0x9888), 0x08170021 },
+	{ _MMIO(0x9888), 0x10170000 },
+	{ _MMIO(0x9888), 0x0633c000 },
+	{ _MMIO(0x9888), 0x0833c000 },
+	{ _MMIO(0x9888), 0x06370800 },
+	{ _MMIO(0x9888), 0x08370840 },
+	{ _MMIO(0x9888), 0x10370000 },
+	{ _MMIO(0x9888), 0x1ace0200 },
+	{ _MMIO(0x9888), 0x0aec5300 },
+	{ _MMIO(0x9888), 0x10ec0000 },
+	{ _MMIO(0x9888), 0x1cec0000 },
+	{ _MMIO(0x9888), 0x0a9b8000 },
+	{ _MMIO(0x9888), 0x1c9c0002 },
+	{ _MMIO(0x9888), 0x0ccc0002 },
+	{ _MMIO(0x9888), 0x0a8d8000 },
+	{ _MMIO(0x9888), 0x108f0001 },
+	{ _MMIO(0x9888), 0x16ac8000 },
+	{ _MMIO(0x9888), 0x0d933031 },
+	{ _MMIO(0x9888), 0x0f933e3f },
+	{ _MMIO(0x9888), 0x01933d00 },
+	{ _MMIO(0x9888), 0x0393073c },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1d930000 },
+	{ _MMIO(0x9888), 0x19930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190003f },
+	{ _MMIO(0x9888), 0x51902240 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x55900242 },
+	{ _MMIO(0x9888), 0x45900084 },
+	{ _MMIO(0x9888), 0x47901400 },
+	{ _MMIO(0x9888), 0x57902220 },
+	{ _MMIO(0x9888), 0x49900c60 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900002 },
+	{ _MMIO(0x9888), 0x43900c63 },
+	{ _MMIO(0x9888), 0x53902222 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x1a4e0820 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x064f0900 },
+	{ _MMIO(0x9888), 0x084f0032 },
+	{ _MMIO(0x9888), 0x0a4f1891 },
+	{ _MMIO(0x9888), 0x0c4f0e00 },
+	{ _MMIO(0x9888), 0x0e4f003c },
+	{ _MMIO(0x9888), 0x004f0d80 },
+	{ _MMIO(0x9888), 0x024f003b },
+	{ _MMIO(0x9888), 0x006c0002 },
+	{ _MMIO(0x9888), 0x086c0100 },
+	{ _MMIO(0x9888), 0x0c6c000c },
+	{ _MMIO(0x9888), 0x0e6c0b00 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x081b8000 },
+	{ _MMIO(0x9888), 0x0c1b4000 },
+	{ _MMIO(0x9888), 0x0e1b8000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1c8000 },
+	{ _MMIO(0x9888), 0x1c1c0024 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5bc000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x1a5c6000 },
+	{ _MMIO(0x9888), 0x1c5c001b },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2000 },
+	{ _MMIO(0x9888), 0x0c4c0208 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020d2000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2cc000 },
+	{ _MMIO(0x9888), 0x162cfb00 },
+	{ _MMIO(0x9888), 0x182c00be },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x19900157 },
+	{ _MMIO(0x9888), 0x1b900158 },
+	{ _MMIO(0x9888), 0x1d900105 },
+	{ _MMIO(0x9888), 0x1f900103 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x11900fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900821 },
+	{ _MMIO(0x9888), 0x47900802 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900802 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900002 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900422 },
+	{ _MMIO(0x9888), 0x53904444 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0c0e001f },
+	{ _MMIO(0x9888), 0x0a0f0000 },
+	{ _MMIO(0x9888), 0x10116800 },
+	{ _MMIO(0x9888), 0x178a03e0 },
+	{ _MMIO(0x9888), 0x11824c00 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x13840020 },
+	{ _MMIO(0x9888), 0x11850019 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x01870c40 },
+	{ _MMIO(0x9888), 0x17880000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x040d4000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020e5400 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x080f0040 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x0e0f0040 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06110012 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x01898000 },
+	{ _MMIO(0x9888), 0x0d890100 },
+	{ _MMIO(0x9888), 0x03898000 },
+	{ _MMIO(0x9888), 0x09808000 },
+	{ _MMIO(0x9888), 0x0b808000 },
+	{ _MMIO(0x9888), 0x0380c000 },
+	{ _MMIO(0x9888), 0x0f8a0075 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x118a8000 },
+	{ _MMIO(0x9888), 0x1b8a4000 },
+	{ _MMIO(0x9888), 0x138a8000 },
+	{ _MMIO(0x9888), 0x1d81a000 },
+	{ _MMIO(0x9888), 0x15818000 },
+	{ _MMIO(0x9888), 0x17818000 },
+	{ _MMIO(0x9888), 0x0b820030 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x0d824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x05824000 },
+	{ _MMIO(0x9888), 0x0d830003 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x03838000 },
+	{ _MMIO(0x9888), 0x07838000 },
+	{ _MMIO(0x9888), 0x0b840980 },
+	{ _MMIO(0x9888), 0x03844d80 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x09850080 },
+	{ _MMIO(0x9888), 0x03850003 },
+	{ _MMIO(0x9888), 0x01850000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x09870032 },
+	{ _MMIO(0x9888), 0x01888052 },
+	{ _MMIO(0x9888), 0x11880000 },
+	{ _MMIO(0x9888), 0x09884000 },
+	{ _MMIO(0x9888), 0x1b931001 },
+	{ _MMIO(0x9888), 0x1d930001 },
+	{ _MMIO(0x9888), 0x19934000 },
+	{ _MMIO(0x9888), 0x1b958000 },
+	{ _MMIO(0x9888), 0x1d950094 },
+	{ _MMIO(0x9888), 0x19958000 },
+	{ _MMIO(0x9888), 0x09e58000 },
+	{ _MMIO(0x9888), 0x0be58000 },
+	{ _MMIO(0x9888), 0x03e5c000 },
+	{ _MMIO(0x9888), 0x0592c000 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d924000 },
+	{ _MMIO(0x9888), 0x0f924000 },
+	{ _MMIO(0x9888), 0x11928000 },
+	{ _MMIO(0x9888), 0x1392c000 },
+	{ _MMIO(0x9888), 0x09924000 },
+	{ _MMIO(0x9888), 0x01985000 },
+	{ _MMIO(0x9888), 0x07988000 },
+	{ _MMIO(0x9888), 0x09981000 },
+	{ _MMIO(0x9888), 0x0b982000 },
+	{ _MMIO(0x9888), 0x0d982000 },
+	{ _MMIO(0x9888), 0x0f989000 },
+	{ _MMIO(0x9888), 0x05982000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1190c080 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900440 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900c21 },
+	{ _MMIO(0x9888), 0x57900400 },
+	{ _MMIO(0x9888), 0x49900042 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900024 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900841 },
+	{ _MMIO(0x9888), 0x53900400 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900064 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900150 },
+	{ _MMIO(0x9888), 0x21900151 },
+	{ _MMIO(0x9888), 0x23900152 },
+	{ _MMIO(0x9888), 0x25900153 },
+	{ _MMIO(0x9888), 0x27900154 },
+	{ _MMIO(0x9888), 0x29900155 },
+	{ _MMIO(0x9888), 0x2b900156 },
+	{ _MMIO(0x9888), 0x2d900157 },
+	{ _MMIO(0x9888), 0x2f90015f },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900160 },
+	{ _MMIO(0x9888), 0x21900161 },
+	{ _MMIO(0x9888), 0x23900162 },
+	{ _MMIO(0x9888), 0x25900163 },
+	{ _MMIO(0x9888), 0x27900164 },
+	{ _MMIO(0x9888), 0x29900165 },
+	{ _MMIO(0x9888), 0x2b900166 },
+	{ _MMIO(0x9888), 0x2d900167 },
+	{ _MMIO(0x9888), 0x2f900150 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended[] = {
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x141c8160 },
+	{ _MMIO(0x9888), 0x161c8015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4eaaa0 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0e6c0b01 },
+	{ _MMIO(0x9888), 0x006c0200 },
+	{ _MMIO(0x9888), 0x026c000c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x001c0041 },
+	{ _MMIO(0x9888), 0x061c4200 },
+	{ _MMIO(0x9888), 0x081c4443 },
+	{ _MMIO(0x9888), 0x0a1c4645 },
+	{ _MMIO(0x9888), 0x0c1c7647 },
+	{ _MMIO(0x9888), 0x041c7357 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x101c0000 },
+	{ _MMIO(0x9888), 0x1a1c0000 },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4caa2a },
+	{ _MMIO(0x9888), 0x0c4c02aa },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5515 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x11907fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900802 },
+	{ _MMIO(0x9888), 0x47900842 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900842 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900800 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c0760 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4e8020 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1ce000 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2a00 },
+	{ _MMIO(0x9888), 0x0c4c0280 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f1500 },
+	{ _MMIO(0x9888), 0x100f0140 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x182c00a0 },
+	{ _MMIO(0x9888), 0x03933300 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190030f },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900021 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x53904444 },
+	{ _MMIO(0x9888), 0x43900000 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x106c0232 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x004f1880 },
+	{ _MMIO(0x9888), 0x024f08bb },
+	{ _MMIO(0x9888), 0x044f001b },
+	{ _MMIO(0x9888), 0x046c0100 },
+	{ _MMIO(0x9888), 0x066c000b },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x041b8000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025bc000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x165c8000 },
+	{ _MMIO(0x9888), 0x185c8000 },
+	{ _MMIO(0x9888), 0x0a4c00a0 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x062cc000 },
+	{ _MMIO(0x9888), 0x082cc000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x07830000 },
+	{ _MMIO(0x9888), 0x1d950080 },
+	{ _MMIO(0x9888), 0x13928000 },
+	{ _MMIO(0x9888), 0x0f988000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b900040 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x126c7b40 },
+	{ _MMIO(0x9888), 0x166c0020 },
+	{ _MMIO(0x9888), 0x0a603444 },
+	{ _MMIO(0x9888), 0x0a613400 },
+	{ _MMIO(0x9888), 0x1a4ea800 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0800 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x1c1c003c },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x10600000 },
+	{ _MMIO(0x9888), 0x04600000 },
+	{ _MMIO(0x9888), 0x0c610044 },
+	{ _MMIO(0x9888), 0x10610000 },
+	{ _MMIO(0x9888), 0x06610000 },
+	{ _MMIO(0x9888), 0x0c4c02a8 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0154 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190ffc0 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900021 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900400 },
+	{ _MMIO(0x9888), 0x43900421 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x126c02e0 },
+	{ _MMIO(0x9888), 0x146c0001 },
+	{ _MMIO(0x9888), 0x0a623400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x026c3324 },
+	{ _MMIO(0x9888), 0x046c3422 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x06614000 },
+	{ _MMIO(0x9888), 0x0c620044 },
+	{ _MMIO(0x9888), 0x10620000 },
+	{ _MMIO(0x9888), 0x06620000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x126c4e80 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x0a633400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x026c3321 },
+	{ _MMIO(0x9888), 0x046c342f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c2000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x06604000 },
+	{ _MMIO(0x9888), 0x0c630044 },
+	{ _MMIO(0x9888), 0x10630000 },
+	{ _MMIO(0x9888), 0x06630000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c00aa },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102f3800 },
+	{ _MMIO(0x9888), 0x144d0500 },
+	{ _MMIO(0x9888), 0x120d03c0 },
+	{ _MMIO(0x9888), 0x140d03cf },
+	{ _MMIO(0x9888), 0x0c0f0004 },
+	{ _MMIO(0x9888), 0x0c4e4000 },
+	{ _MMIO(0x9888), 0x042f0480 },
+	{ _MMIO(0x9888), 0x082f0000 },
+	{ _MMIO(0x9888), 0x022f0000 },
+	{ _MMIO(0x9888), 0x0a4c0090 },
+	{ _MMIO(0x9888), 0x064d0027 },
+	{ _MMIO(0x9888), 0x004d0000 },
+	{ _MMIO(0x9888), 0x000d0d40 },
+	{ _MMIO(0x9888), 0x020d803f },
+	{ _MMIO(0x9888), 0x040d8023 },
+	{ _MMIO(0x9888), 0x100d0000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020f0010 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x0e0f0050 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41901400 },
+	{ _MMIO(0x9888), 0x43901485 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x14152c00 },
+	{ _MMIO(0x9888), 0x16150005 },
+	{ _MMIO(0x9888), 0x121600a0 },
+	{ _MMIO(0x9888), 0x14352c00 },
+	{ _MMIO(0x9888), 0x16350005 },
+	{ _MMIO(0x9888), 0x123600a0 },
+	{ _MMIO(0x9888), 0x14552c00 },
+	{ _MMIO(0x9888), 0x16550005 },
+	{ _MMIO(0x9888), 0x125600a0 },
+	{ _MMIO(0x9888), 0x062f6000 },
+	{ _MMIO(0x9888), 0x022f2000 },
+	{ _MMIO(0x9888), 0x0c4c0050 },
+	{ _MMIO(0x9888), 0x0a4c0010 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0350 },
+	{ _MMIO(0x9888), 0x0c0fb000 },
+	{ _MMIO(0x9888), 0x0e0f00da },
+	{ _MMIO(0x9888), 0x182c0028 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x022dc000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x0c138000 },
+	{ _MMIO(0x9888), 0x0e132000 },
+	{ _MMIO(0x9888), 0x0413c000 },
+	{ _MMIO(0x9888), 0x1c140018 },
+	{ _MMIO(0x9888), 0x0c157000 },
+	{ _MMIO(0x9888), 0x0e150078 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04162180 },
+	{ _MMIO(0x9888), 0x02160000 },
+	{ _MMIO(0x9888), 0x04174000 },
+	{ _MMIO(0x9888), 0x0233a000 },
+	{ _MMIO(0x9888), 0x04333000 },
+	{ _MMIO(0x9888), 0x14348000 },
+	{ _MMIO(0x9888), 0x16348000 },
+	{ _MMIO(0x9888), 0x02357870 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04360043 },
+	{ _MMIO(0x9888), 0x02360000 },
+	{ _MMIO(0x9888), 0x04371000 },
+	{ _MMIO(0x9888), 0x0e538000 },
+	{ _MMIO(0x9888), 0x00538000 },
+	{ _MMIO(0x9888), 0x06533000 },
+	{ _MMIO(0x9888), 0x1c540020 },
+	{ _MMIO(0x9888), 0x12548000 },
+	{ _MMIO(0x9888), 0x0e557000 },
+	{ _MMIO(0x9888), 0x00557800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06560043 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x06571000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900060 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900060 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x12120000 },
+	{ _MMIO(0x9888), 0x12320000 },
+	{ _MMIO(0x9888), 0x12520000 },
+	{ _MMIO(0x9888), 0x002f8000 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0015 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f03a0 },
+	{ _MMIO(0x9888), 0x0c0ff000 },
+	{ _MMIO(0x9888), 0x0e0f0095 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x02108000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x02118000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x02121880 },
+	{ _MMIO(0x9888), 0x041219b5 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x02134000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x0c308000 },
+	{ _MMIO(0x9888), 0x0e304000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x0c318000 },
+	{ _MMIO(0x9888), 0x0e314000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x0c321a80 },
+	{ _MMIO(0x9888), 0x0e320033 },
+	{ _MMIO(0x9888), 0x06320031 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x0c334000 },
+	{ _MMIO(0x9888), 0x0e331000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0e508000 },
+	{ _MMIO(0x9888), 0x00508000 },
+	{ _MMIO(0x9888), 0x02504000 },
+	{ _MMIO(0x9888), 0x0e518000 },
+	{ _MMIO(0x9888), 0x00518000 },
+	{ _MMIO(0x9888), 0x02514000 },
+	{ _MMIO(0x9888), 0x0e521880 },
+	{ _MMIO(0x9888), 0x00521a80 },
+	{ _MMIO(0x9888), 0x02520033 },
+	{ _MMIO(0x9888), 0x0e534000 },
+	{ _MMIO(0x9888), 0x00534000 },
+	{ _MMIO(0x9888), 0x02531000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900062 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x12124d60 },
+	{ _MMIO(0x9888), 0x12322e60 },
+	{ _MMIO(0x9888), 0x12524d60 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0014 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0fe000 },
+	{ _MMIO(0x9888), 0x0e0f0097 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x002d8000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x04121fb7 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x00308000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x00318000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x00321b80 },
+	{ _MMIO(0x9888), 0x0632003f },
+	{ _MMIO(0x9888), 0x00334000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0250c000 },
+	{ _MMIO(0x9888), 0x0251c000 },
+	{ _MMIO(0x9888), 0x02521fb7 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x02535000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900063 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x121203e0 },
+	{ _MMIO(0x9888), 0x123203e0 },
+	{ _MMIO(0x9888), 0x125203e0 },
+	{ _MMIO(0x9888), 0x129203e0 },
+	{ _MMIO(0x9888), 0x12b203e0 },
+	{ _MMIO(0x9888), 0x12d203e0 },
+	{ _MMIO(0x9888), 0x024ec000 },
+	{ _MMIO(0x9888), 0x044ec000 },
+	{ _MMIO(0x9888), 0x064ec000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c0042 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f006d },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x042d8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06114000 },
+	{ _MMIO(0x9888), 0x06120033 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x04308000 },
+	{ _MMIO(0x9888), 0x04318000 },
+	{ _MMIO(0x9888), 0x04321980 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x04334000 },
+	{ _MMIO(0x9888), 0x04504000 },
+	{ _MMIO(0x9888), 0x04514000 },
+	{ _MMIO(0x9888), 0x04520033 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x04531000 },
+	{ _MMIO(0x9888), 0x00af8000 },
+	{ _MMIO(0x9888), 0x0acc0001 },
+	{ _MMIO(0x9888), 0x008d8000 },
+	{ _MMIO(0x9888), 0x028da000 },
+	{ _MMIO(0x9888), 0x0c8fb000 },
+	{ _MMIO(0x9888), 0x0e8f0001 },
+	{ _MMIO(0x9888), 0x06ac8000 },
+	{ _MMIO(0x9888), 0x02ad4000 },
+	{ _MMIO(0x9888), 0x02908000 },
+	{ _MMIO(0x9888), 0x02918000 },
+	{ _MMIO(0x9888), 0x02921980 },
+	{ _MMIO(0x9888), 0x00920000 },
+	{ _MMIO(0x9888), 0x02934000 },
+	{ _MMIO(0x9888), 0x02b04000 },
+	{ _MMIO(0x9888), 0x02b14000 },
+	{ _MMIO(0x9888), 0x02b20033 },
+	{ _MMIO(0x9888), 0x00b20000 },
+	{ _MMIO(0x9888), 0x02b31000 },
+	{ _MMIO(0x9888), 0x00d08000 },
+	{ _MMIO(0x9888), 0x00d18000 },
+	{ _MMIO(0x9888), 0x00d21980 },
+	{ _MMIO(0x9888), 0x00d34000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900002 },
+	{ _MMIO(0x9888), 0x53900420 },
+	{ _MMIO(0x9888), 0x459000a1 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00100030 },
+	{ _MMIO(0x2774), 0x0000fff9 },
+	{ _MMIO(0x2778), 0x00000002 },
+	{ _MMIO(0x277c), 0x0000fffc },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000fff3 },
+	{ _MMIO(0x2788), 0x00100180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000ff3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00008003 },
+};
+
+static const struct i915_oa_reg mux_config_vme_pipe[] = {
+	{ _MMIO(0x9888), 0x141a5800 },
+	{ _MMIO(0x9888), 0x161a00c0 },
+	{ _MMIO(0x9888), 0x12180240 },
+	{ _MMIO(0x9888), 0x14180002 },
+	{ _MMIO(0x9888), 0x149a5800 },
+	{ _MMIO(0x9888), 0x169a00c0 },
+	{ _MMIO(0x9888), 0x12980240 },
+	{ _MMIO(0x9888), 0x14980002 },
+	{ _MMIO(0x9888), 0x1a4e3fc0 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x022f8000 },
+	{ _MMIO(0x9888), 0x042f3000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c9500 },
+	{ _MMIO(0x9888), 0x0c4c002a },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0015 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c000a },
+	{ _MMIO(0x9888), 0x04193000 },
+	{ _MMIO(0x9888), 0x081a28c1 },
+	{ _MMIO(0x9888), 0x001a0000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x0613c000 },
+	{ _MMIO(0x9888), 0x0813f000 },
+	{ _MMIO(0x9888), 0x00172000 },
+	{ _MMIO(0x9888), 0x06178000 },
+	{ _MMIO(0x9888), 0x0817a000 },
+	{ _MMIO(0x9888), 0x00180037 },
+	{ _MMIO(0x9888), 0x06180940 },
+	{ _MMIO(0x9888), 0x08180000 },
+	{ _MMIO(0x9888), 0x02180000 },
+	{ _MMIO(0x9888), 0x04183000 },
+	{ _MMIO(0x9888), 0x04afc000 },
+	{ _MMIO(0x9888), 0x06af3000 },
+	{ _MMIO(0x9888), 0x0acc4000 },
+	{ _MMIO(0x9888), 0x0ccc0015 },
+	{ _MMIO(0x9888), 0x0a8da000 },
+	{ _MMIO(0x9888), 0x0c8da000 },
+	{ _MMIO(0x9888), 0x0e8f4000 },
+	{ _MMIO(0x9888), 0x108f0015 },
+	{ _MMIO(0x9888), 0x16aca000 },
+	{ _MMIO(0x9888), 0x18ac000a },
+	{ _MMIO(0x9888), 0x06993000 },
+	{ _MMIO(0x9888), 0x0c9a28c1 },
+	{ _MMIO(0x9888), 0x009a0000 },
+	{ _MMIO(0x9888), 0x0a93f000 },
+	{ _MMIO(0x9888), 0x0c93f000 },
+	{ _MMIO(0x9888), 0x0a97a000 },
+	{ _MMIO(0x9888), 0x0c97a000 },
+	{ _MMIO(0x9888), 0x0a980977 },
+	{ _MMIO(0x9888), 0x08980000 },
+	{ _MMIO(0x9888), 0x04980000 },
+	{ _MMIO(0x9888), 0x06983000 },
+	{ _MMIO(0x9888), 0x119000ff },
+	{ _MMIO(0x9888), 0x51900040 },
+	{ _MMIO(0x9888), 0x41900020 },
+	{ _MMIO(0x9888), 0x55900004 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x479008a5 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900002 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
+			const struct i915_oa_reg **regs,
+			int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_vme_pipe;
+	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x11810000 },
+	{ _MMIO(0x9888), 0x07810013 },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930040 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_kblgt3(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_VME_PIPE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_vme_pipe_mux_config(dev_priv,
+						dev_priv->perf.oa.mux_regs,
+						dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_vme_pipe;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_vme_pipe);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_vme_pipe;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_vme_pipe);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "0286c920-2f6d-493b-b22d-7a5280df43de",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "9823aaa1-b06f-40ce-884b-cd798c79f0c2",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "c7c735f3-ce58-45cf-aa04-30b183f1faff",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "96ec2219-040b-428a-856a-6bc03363a057",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "03372b64-4996-4d3b-aa18-790e75eeb9c2",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "31b4ce5a-bd61-4c1f-bb5d-f2e731412150",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "2ce0911a-27fc-4887-96f0-11084fa807c3",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "546c4c1d-99b8-42fb-a107-5aaabb5314a8",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "4e93d156-9b39-4268-8544-a8e0480806d7",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "de1bec86-ca92-4b43-89fa-147653221cc0",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "e63537bb-10be-4d4a-92c4-c6b0c65e02ef",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "7a03a9f8-ec5e-46bb-8b67-1f0ff1476281",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "b25d2ebf-a6e0-4b29-96be-a9b010edeeda",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "469a05e5-e299-46f7-9598-7b05f3c34991",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "52f925c6-786a-4ec6-86ce-cba85c83453a",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "efc497ac-884e-4ee4-a4a8-15fba22aaf21",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
+}
+
+static struct device_attribute dev_attr_vme_pipe_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_vme_pipe_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_vme_pipe[] = {
+	&dev_attr_vme_pipe_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_vme_pipe = {
+	.name = "bfd9764d-2c5b-4c16-bfc1-89de3ca10917",
+	.attrs =  attrs_vme_pipe,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "f1792f32-6db2-4b50-b4b2-557128f1688d",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_kblgt3(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+		if (ret)
+			goto error_vme_pipe;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+error_vme_pipe:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_kblgt3(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.h b/drivers/gpu/drm/i915/i915_oa_kblgt3.h
new file mode 100644
index 000000000000..b0ca7f3114d3
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_KBLGT3_H__
+#define __I915_OA_KBLGT3_H__
+
+extern int i915_oa_n_builtin_metric_sets_kblgt3;
+
+extern int i915_oa_select_metric_set_kblgt3(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_kblgt3(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_kblgt3(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.c b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
new file mode 100644
index 000000000000..1268beda212c
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
@@ -0,0 +1,3479 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_sklgt2.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_VME_PIPE,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_sklgt2 = 18;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic_1_sku_gte_0x02[] = {
+	{ _MMIO(0x9888), 0x166c01e0 },
+	{ _MMIO(0x9888), 0x12170280 },
+	{ _MMIO(0x9888), 0x12370280 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x1a4e0080 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x0a1b4000 },
+	{ _MMIO(0x9888), 0x1c1c0001 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x042f1000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c8400 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0d2000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f6600 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x162c2200 },
+	{ _MMIO(0x9888), 0x062d8000 },
+	{ _MMIO(0x9888), 0x082d8000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x08133000 },
+	{ _MMIO(0x9888), 0x00170020 },
+	{ _MMIO(0x9888), 0x08170021 },
+	{ _MMIO(0x9888), 0x10170000 },
+	{ _MMIO(0x9888), 0x0633c000 },
+	{ _MMIO(0x9888), 0x0833c000 },
+	{ _MMIO(0x9888), 0x06370800 },
+	{ _MMIO(0x9888), 0x08370840 },
+	{ _MMIO(0x9888), 0x10370000 },
+	{ _MMIO(0x9888), 0x0d933031 },
+	{ _MMIO(0x9888), 0x0f933e3f },
+	{ _MMIO(0x9888), 0x01933d00 },
+	{ _MMIO(0x9888), 0x0393073c },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1d930000 },
+	{ _MMIO(0x9888), 0x19930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190001f },
+	{ _MMIO(0x9888), 0x51904400 },
+	{ _MMIO(0x9888), 0x41900020 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c21 },
+	{ _MMIO(0x9888), 0x47900061 },
+	{ _MMIO(0x9888), 0x57904440 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900004 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53904444 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	if (dev_priv->drm.pdev->revision >= 0x02) {
+		regs[n] = mux_config_render_basic_1_sku_gte_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_render_basic_1_sku_gte_0x02);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic_0_slices_0x01_and_sku_lt_0x02[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901403 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4e8200 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x004f0db2 },
+	{ _MMIO(0x9888), 0x064f0900 },
+	{ _MMIO(0x9888), 0x084f1880 },
+	{ _MMIO(0x9888), 0x0a4f0011 },
+	{ _MMIO(0x9888), 0x0c4f0e3c },
+	{ _MMIO(0x9888), 0x0e4f1d80 },
+	{ _MMIO(0x9888), 0x086c0002 },
+	{ _MMIO(0x9888), 0x0a6c0100 },
+	{ _MMIO(0x9888), 0x0e6c000c },
+	{ _MMIO(0x9888), 0x026c000b },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x081b4000 },
+	{ _MMIO(0x9888), 0x0a1b8000 },
+	{ _MMIO(0x9888), 0x0e1b4000 },
+	{ _MMIO(0x9888), 0x021b4000 },
+	{ _MMIO(0x9888), 0x1a1c4000 },
+	{ _MMIO(0x9888), 0x1c1c0012 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x005bc000 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b8000 },
+	{ _MMIO(0x9888), 0x0a5b4000 },
+	{ _MMIO(0x9888), 0x0c5bc000 },
+	{ _MMIO(0x9888), 0x0e5b8000 },
+	{ _MMIO(0x9888), 0x105c8000 },
+	{ _MMIO(0x9888), 0x1a5ca000 },
+	{ _MMIO(0x9888), 0x1c5c002d },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x0a4c0800 },
+	{ _MMIO(0x9888), 0x0c4c0082 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020d2000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002cc000 },
+	{ _MMIO(0x9888), 0x0e2cc000 },
+	{ _MMIO(0x9888), 0x162cbe00 },
+	{ _MMIO(0x9888), 0x182c00ef },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x19900157 },
+	{ _MMIO(0x9888), 0x1b900167 },
+	{ _MMIO(0x9888), 0x1d900105 },
+	{ _MMIO(0x9888), 0x1f900103 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0xd28), 0x00000000 },
+	{ _MMIO(0x9888), 0x11900fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900840 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900842 },
+	{ _MMIO(0x9888), 0x47900840 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900840 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900040 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900840 },
+	{ _MMIO(0x9888), 0x53901111 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic_0_slices_0x01_and_sku_gte_0x02[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901403 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x1a4e0820 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x064f0900 },
+	{ _MMIO(0x9888), 0x084f0032 },
+	{ _MMIO(0x9888), 0x0a4f1810 },
+	{ _MMIO(0x9888), 0x0c4f0e00 },
+	{ _MMIO(0x9888), 0x0e4f003c },
+	{ _MMIO(0x9888), 0x004f0d80 },
+	{ _MMIO(0x9888), 0x024f003b },
+	{ _MMIO(0x9888), 0x006c0002 },
+	{ _MMIO(0x9888), 0x086c0000 },
+	{ _MMIO(0x9888), 0x0c6c000c },
+	{ _MMIO(0x9888), 0x0e6c0b00 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x081b8000 },
+	{ _MMIO(0x9888), 0x0c1b4000 },
+	{ _MMIO(0x9888), 0x0e1b8000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1c8000 },
+	{ _MMIO(0x9888), 0x1c1c0024 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5bc000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x1a5c6000 },
+	{ _MMIO(0x9888), 0x1c5c001b },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2000 },
+	{ _MMIO(0x9888), 0x0c4c0208 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020d2000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2cc000 },
+	{ _MMIO(0x9888), 0x162cfb00 },
+	{ _MMIO(0x9888), 0x182c00be },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x19900157 },
+	{ _MMIO(0x9888), 0x1b900167 },
+	{ _MMIO(0x9888), 0x1d900105 },
+	{ _MMIO(0x9888), 0x1f900103 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x11900fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900842 },
+	{ _MMIO(0x9888), 0x47900802 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900802 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900002 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53901111 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
+
+	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
+	    (dev_priv->drm.pdev->revision < 0x02)) {
+		regs[n] = mux_config_compute_basic_0_slices_0x01_and_sku_lt_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_compute_basic_0_slices_0x01_and_sku_lt_0x02);
+		n++;
+	}
+	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
+		   (dev_priv->drm.pdev->revision >= 0x02)) {
+		regs[n] = mux_config_compute_basic_0_slices_0x01_and_sku_gte_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_compute_basic_0_slices_0x01_and_sku_gte_0x02);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile_0_sku_lt_0x02[] = {
+	{ _MMIO(0x9888), 0x0c0e001f },
+	{ _MMIO(0x9888), 0x0a0f0000 },
+	{ _MMIO(0x9888), 0x10116800 },
+	{ _MMIO(0x9888), 0x178a03e0 },
+	{ _MMIO(0x9888), 0x11824c00 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x13840020 },
+	{ _MMIO(0x9888), 0x11850019 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x01870c40 },
+	{ _MMIO(0x9888), 0x17880000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x040d4000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020e5400 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x080f0040 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x0e0f0040 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06110012 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x01898000 },
+	{ _MMIO(0x9888), 0x0d890100 },
+	{ _MMIO(0x9888), 0x03898000 },
+	{ _MMIO(0x9888), 0x09808000 },
+	{ _MMIO(0x9888), 0x0b808000 },
+	{ _MMIO(0x9888), 0x0380c000 },
+	{ _MMIO(0x9888), 0x0f8a0075 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x118a8000 },
+	{ _MMIO(0x9888), 0x1b8a4000 },
+	{ _MMIO(0x9888), 0x138a8000 },
+	{ _MMIO(0x9888), 0x1d81a000 },
+	{ _MMIO(0x9888), 0x15818000 },
+	{ _MMIO(0x9888), 0x17818000 },
+	{ _MMIO(0x9888), 0x0b820030 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x0d824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x05824000 },
+	{ _MMIO(0x9888), 0x0d830003 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x03838000 },
+	{ _MMIO(0x9888), 0x07838000 },
+	{ _MMIO(0x9888), 0x0b840980 },
+	{ _MMIO(0x9888), 0x03844d80 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x09850080 },
+	{ _MMIO(0x9888), 0x03850003 },
+	{ _MMIO(0x9888), 0x01850000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x09870032 },
+	{ _MMIO(0x9888), 0x01888052 },
+	{ _MMIO(0x9888), 0x11880000 },
+	{ _MMIO(0x9888), 0x09884000 },
+	{ _MMIO(0x9888), 0x15968000 },
+	{ _MMIO(0x9888), 0x17968000 },
+	{ _MMIO(0x9888), 0x0f96c000 },
+	{ _MMIO(0x9888), 0x1f950011 },
+	{ _MMIO(0x9888), 0x1d950014 },
+	{ _MMIO(0x9888), 0x0592c000 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d924000 },
+	{ _MMIO(0x9888), 0x0f924000 },
+	{ _MMIO(0x9888), 0x11928000 },
+	{ _MMIO(0x9888), 0x1392c000 },
+	{ _MMIO(0x9888), 0x09924000 },
+	{ _MMIO(0x9888), 0x01985000 },
+	{ _MMIO(0x9888), 0x07988000 },
+	{ _MMIO(0x9888), 0x09981000 },
+	{ _MMIO(0x9888), 0x0b982000 },
+	{ _MMIO(0x9888), 0x0d982000 },
+	{ _MMIO(0x9888), 0x0f989000 },
+	{ _MMIO(0x9888), 0x05982000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x0b978000 },
+	{ _MMIO(0x9888), 0x0f974000 },
+	{ _MMIO(0x9888), 0x11974000 },
+	{ _MMIO(0x9888), 0x13978000 },
+	{ _MMIO(0x9888), 0x09974000 },
+	{ _MMIO(0xd28), 0x00000000 },
+	{ _MMIO(0x9888), 0x1190c080 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x419010a0 },
+	{ _MMIO(0x9888), 0x55904000 },
+	{ _MMIO(0x9888), 0x45901000 },
+	{ _MMIO(0x9888), 0x47900084 },
+	{ _MMIO(0x9888), 0x57904400 },
+	{ _MMIO(0x9888), 0x499000a5 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900081 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x439014a4 },
+	{ _MMIO(0x9888), 0x53900400 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile_0_sku_gte_0x02[] = {
+	{ _MMIO(0x9888), 0x0c0e001f },
+	{ _MMIO(0x9888), 0x0a0f0000 },
+	{ _MMIO(0x9888), 0x10116800 },
+	{ _MMIO(0x9888), 0x178a03e0 },
+	{ _MMIO(0x9888), 0x11824c00 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x13840020 },
+	{ _MMIO(0x9888), 0x11850019 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x01870c40 },
+	{ _MMIO(0x9888), 0x17880000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x040d4000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020e5400 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x080f0040 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x0e0f0040 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06110012 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x01898000 },
+	{ _MMIO(0x9888), 0x0d890100 },
+	{ _MMIO(0x9888), 0x03898000 },
+	{ _MMIO(0x9888), 0x09808000 },
+	{ _MMIO(0x9888), 0x0b808000 },
+	{ _MMIO(0x9888), 0x0380c000 },
+	{ _MMIO(0x9888), 0x0f8a0075 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x118a8000 },
+	{ _MMIO(0x9888), 0x1b8a4000 },
+	{ _MMIO(0x9888), 0x138a8000 },
+	{ _MMIO(0x9888), 0x1d81a000 },
+	{ _MMIO(0x9888), 0x15818000 },
+	{ _MMIO(0x9888), 0x17818000 },
+	{ _MMIO(0x9888), 0x0b820030 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x0d824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x05824000 },
+	{ _MMIO(0x9888), 0x0d830003 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x03838000 },
+	{ _MMIO(0x9888), 0x07838000 },
+	{ _MMIO(0x9888), 0x0b840980 },
+	{ _MMIO(0x9888), 0x03844d80 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x09850080 },
+	{ _MMIO(0x9888), 0x03850003 },
+	{ _MMIO(0x9888), 0x01850000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x09870032 },
+	{ _MMIO(0x9888), 0x01888052 },
+	{ _MMIO(0x9888), 0x11880000 },
+	{ _MMIO(0x9888), 0x09884000 },
+	{ _MMIO(0x9888), 0x1b931001 },
+	{ _MMIO(0x9888), 0x1d930001 },
+	{ _MMIO(0x9888), 0x19934000 },
+	{ _MMIO(0x9888), 0x1b958000 },
+	{ _MMIO(0x9888), 0x1d950094 },
+	{ _MMIO(0x9888), 0x19958000 },
+	{ _MMIO(0x9888), 0x05e5a000 },
+	{ _MMIO(0x9888), 0x01e5c000 },
+	{ _MMIO(0x9888), 0x0592c000 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d924000 },
+	{ _MMIO(0x9888), 0x0f924000 },
+	{ _MMIO(0x9888), 0x11928000 },
+	{ _MMIO(0x9888), 0x1392c000 },
+	{ _MMIO(0x9888), 0x09924000 },
+	{ _MMIO(0x9888), 0x01985000 },
+	{ _MMIO(0x9888), 0x07988000 },
+	{ _MMIO(0x9888), 0x09981000 },
+	{ _MMIO(0x9888), 0x0b982000 },
+	{ _MMIO(0x9888), 0x0d982000 },
+	{ _MMIO(0x9888), 0x0f989000 },
+	{ _MMIO(0x9888), 0x05982000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1190c080 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x419010a0 },
+	{ _MMIO(0x9888), 0x55904000 },
+	{ _MMIO(0x9888), 0x45901000 },
+	{ _MMIO(0x9888), 0x47900084 },
+	{ _MMIO(0x9888), 0x57904400 },
+	{ _MMIO(0x9888), 0x499000a5 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900081 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x439014a4 },
+	{ _MMIO(0x9888), 0x53900400 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
+
+	if (dev_priv->drm.pdev->revision < 0x02) {
+		regs[n] = mux_config_render_pipe_profile_0_sku_lt_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile_0_sku_lt_0x02);
+		n++;
+	}
+	if (dev_priv->drm.pdev->revision >= 0x02) {
+		regs[n] = mux_config_render_pipe_profile_0_sku_gte_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile_0_sku_gte_0x02);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads_0_slices_0x01_and_sku_lt_0x02[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x13946000 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x0f968000 },
+	{ _MMIO(0x9888), 0x1196c000 },
+	{ _MMIO(0x9888), 0x13964000 },
+	{ _MMIO(0x9888), 0x11938000 },
+	{ _MMIO(0x9888), 0x1b93fe00 },
+	{ _MMIO(0x9888), 0x01940010 },
+	{ _MMIO(0x9888), 0x07941100 },
+	{ _MMIO(0x9888), 0x09941312 },
+	{ _MMIO(0x9888), 0x0b941514 },
+	{ _MMIO(0x9888), 0x0d941716 },
+	{ _MMIO(0x9888), 0x11940000 },
+	{ _MMIO(0x9888), 0x19940000 },
+	{ _MMIO(0x9888), 0x1b940000 },
+	{ _MMIO(0x9888), 0x1d940000 },
+	{ _MMIO(0x9888), 0x1b954000 },
+	{ _MMIO(0x9888), 0x1d95a550 },
+	{ _MMIO(0x9888), 0x1f9502aa },
+	{ _MMIO(0x9888), 0x2f900157 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13908000 },
+	{ _MMIO(0x9888), 0x21908000 },
+	{ _MMIO(0x9888), 0x23908000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27908000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0xd28), 0x00000000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads_0_sku_lt_0x05_and_sku_gte_0x02[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x13946000 },
+	{ _MMIO(0x9888), 0x15940016 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x19930800 },
+	{ _MMIO(0x9888), 0x1b93aa55 },
+	{ _MMIO(0x9888), 0x1d9300aa },
+	{ _MMIO(0x9888), 0x01940010 },
+	{ _MMIO(0x9888), 0x07941100 },
+	{ _MMIO(0x9888), 0x09941312 },
+	{ _MMIO(0x9888), 0x0b941514 },
+	{ _MMIO(0x9888), 0x0d941716 },
+	{ _MMIO(0x9888), 0x0f940018 },
+	{ _MMIO(0x9888), 0x1b940000 },
+	{ _MMIO(0x9888), 0x11940000 },
+	{ _MMIO(0x9888), 0x01e58000 },
+	{ _MMIO(0x9888), 0x03e57000 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13908000 },
+	{ _MMIO(0x9888), 0x21908000 },
+	{ _MMIO(0x9888), 0x23908000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27908000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c20 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900421 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900421 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900061 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads_0_sku_gte_0x05[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900064 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900150 },
+	{ _MMIO(0x9888), 0x21900151 },
+	{ _MMIO(0x9888), 0x23900152 },
+	{ _MMIO(0x9888), 0x25900153 },
+	{ _MMIO(0x9888), 0x27900154 },
+	{ _MMIO(0x9888), 0x29900155 },
+	{ _MMIO(0x9888), 0x2b900156 },
+	{ _MMIO(0x9888), 0x2d900157 },
+	{ _MMIO(0x9888), 0x2f90015f },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 3);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 3);
+
+	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
+	    (dev_priv->drm.pdev->revision < 0x02)) {
+		regs[n] = mux_config_memory_reads_0_slices_0x01_and_sku_lt_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_memory_reads_0_slices_0x01_and_sku_lt_0x02);
+		n++;
+	}
+	if ((dev_priv->drm.pdev->revision < 0x05) &&
+		   (dev_priv->drm.pdev->revision >= 0x02)) {
+		regs[n] = mux_config_memory_reads_0_sku_lt_0x05_and_sku_gte_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_memory_reads_0_sku_lt_0x05_and_sku_gte_0x02);
+		n++;
+	}
+	if (dev_priv->drm.pdev->revision >= 0x05) {
+		regs[n] = mux_config_memory_reads_0_sku_gte_0x05;
+		lens[n] = ARRAY_SIZE(mux_config_memory_reads_0_sku_gte_0x05);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes_0_slices_0x01_and_sku_lt_0x02[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x13945400 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901400 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x0f968000 },
+	{ _MMIO(0x9888), 0x1196c000 },
+	{ _MMIO(0x9888), 0x13964000 },
+	{ _MMIO(0x9888), 0x11938000 },
+	{ _MMIO(0x9888), 0x1b93fe00 },
+	{ _MMIO(0x9888), 0x01940010 },
+	{ _MMIO(0x9888), 0x07941100 },
+	{ _MMIO(0x9888), 0x09941312 },
+	{ _MMIO(0x9888), 0x0b941514 },
+	{ _MMIO(0x9888), 0x0d941716 },
+	{ _MMIO(0x9888), 0x11940000 },
+	{ _MMIO(0x9888), 0x19940000 },
+	{ _MMIO(0x9888), 0x1b940000 },
+	{ _MMIO(0x9888), 0x1d940000 },
+	{ _MMIO(0x9888), 0x1b954000 },
+	{ _MMIO(0x9888), 0x1d95a550 },
+	{ _MMIO(0x9888), 0x1f9502aa },
+	{ _MMIO(0x9888), 0x2f900167 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13908000 },
+	{ _MMIO(0x9888), 0x21908000 },
+	{ _MMIO(0x9888), 0x23908000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27908000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0xd28), 0x00000000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes_0_sku_lt_0x05_and_sku_gte_0x02[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x13945400 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901400 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x19930800 },
+	{ _MMIO(0x9888), 0x1b93aa55 },
+	{ _MMIO(0x9888), 0x1d93002a },
+	{ _MMIO(0x9888), 0x01940010 },
+	{ _MMIO(0x9888), 0x07941100 },
+	{ _MMIO(0x9888), 0x09941312 },
+	{ _MMIO(0x9888), 0x0b941514 },
+	{ _MMIO(0x9888), 0x0d941716 },
+	{ _MMIO(0x9888), 0x1b940000 },
+	{ _MMIO(0x9888), 0x11940000 },
+	{ _MMIO(0x9888), 0x01e58000 },
+	{ _MMIO(0x9888), 0x03e57000 },
+	{ _MMIO(0x9888), 0x2f900167 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x13908000 },
+	{ _MMIO(0x9888), 0x21908000 },
+	{ _MMIO(0x9888), 0x23908000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27908000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c20 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900421 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900421 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes_0_sku_gte_0x05[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900160 },
+	{ _MMIO(0x9888), 0x21900161 },
+	{ _MMIO(0x9888), 0x23900162 },
+	{ _MMIO(0x9888), 0x25900163 },
+	{ _MMIO(0x9888), 0x27900164 },
+	{ _MMIO(0x9888), 0x29900165 },
+	{ _MMIO(0x9888), 0x2b900166 },
+	{ _MMIO(0x9888), 0x2d900167 },
+	{ _MMIO(0x9888), 0x2f900150 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 3);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 3);
+
+	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
+	    (dev_priv->drm.pdev->revision < 0x02)) {
+		regs[n] = mux_config_memory_writes_0_slices_0x01_and_sku_lt_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_memory_writes_0_slices_0x01_and_sku_lt_0x02);
+		n++;
+	}
+	if ((dev_priv->drm.pdev->revision < 0x05) &&
+		   (dev_priv->drm.pdev->revision >= 0x02)) {
+		regs[n] = mux_config_memory_writes_0_sku_lt_0x05_and_sku_gte_0x02;
+		lens[n] = ARRAY_SIZE(mux_config_memory_writes_0_sku_lt_0x05_and_sku_gte_0x02);
+		n++;
+	}
+	if (dev_priv->drm.pdev->revision >= 0x05) {
+		regs[n] = mux_config_memory_writes_0_sku_gte_0x05;
+		lens[n] = ARRAY_SIZE(mux_config_memory_writes_0_sku_gte_0x05);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended_0_subslices_0x01[] = {
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x141c8160 },
+	{ _MMIO(0x9888), 0x161c8015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4eaaa0 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0e6c0b01 },
+	{ _MMIO(0x9888), 0x006c0200 },
+	{ _MMIO(0x9888), 0x026c000c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x001c0041 },
+	{ _MMIO(0x9888), 0x061c4200 },
+	{ _MMIO(0x9888), 0x081c4443 },
+	{ _MMIO(0x9888), 0x0a1c4645 },
+	{ _MMIO(0x9888), 0x0c1c7647 },
+	{ _MMIO(0x9888), 0x041c7357 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x101c0000 },
+	{ _MMIO(0x9888), 0x1a1c0000 },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4caa2a },
+	{ _MMIO(0x9888), 0x0c4c02aa },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5515 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0xd28), 0x00000000 },
+	{ _MMIO(0x9888), 0x11907fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900802 },
+	{ _MMIO(0x9888), 0x47900842 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900842 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900800 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
+		regs[n] = mux_config_compute_extended_0_subslices_0x01;
+		lens[n] = ARRAY_SIZE(mux_config_compute_extended_0_subslices_0x01);
+		n++;
+	}
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c0760 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f901403 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4e8020 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1ce000 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2a00 },
+	{ _MMIO(0x9888), 0x0c4c0280 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f1500 },
+	{ _MMIO(0x9888), 0x100f0140 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x182c00a0 },
+	{ _MMIO(0x9888), 0x03933300 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900167 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190030f },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900042 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x53901111 },
+	{ _MMIO(0x9888), 0x43900420 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x106c0232 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x004f1880 },
+	{ _MMIO(0x9888), 0x024f08bb },
+	{ _MMIO(0x9888), 0x044f001b },
+	{ _MMIO(0x9888), 0x046c0100 },
+	{ _MMIO(0x9888), 0x066c000b },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x041b8000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025bc000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x165c8000 },
+	{ _MMIO(0x9888), 0x185c8000 },
+	{ _MMIO(0x9888), 0x0a4c00a0 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x062cc000 },
+	{ _MMIO(0x9888), 0x082cc000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x07830000 },
+	{ _MMIO(0x9888), 0x1d950080 },
+	{ _MMIO(0x9888), 0x13928000 },
+	{ _MMIO(0x9888), 0x0f988000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x4b9000a0 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x126c7b40 },
+	{ _MMIO(0x9888), 0x166c0020 },
+	{ _MMIO(0x9888), 0x0a603444 },
+	{ _MMIO(0x9888), 0x0a613400 },
+	{ _MMIO(0x9888), 0x1a4ea800 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0800 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x1c1c003c },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x10600000 },
+	{ _MMIO(0x9888), 0x04600000 },
+	{ _MMIO(0x9888), 0x0c610044 },
+	{ _MMIO(0x9888), 0x10610000 },
+	{ _MMIO(0x9888), 0x06610000 },
+	{ _MMIO(0x9888), 0x0c4c02a8 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0154 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190ffc0 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900021 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900400 },
+	{ _MMIO(0x9888), 0x43900421 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x126c02e0 },
+	{ _MMIO(0x9888), 0x146c0001 },
+	{ _MMIO(0x9888), 0x0a623400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x026c3324 },
+	{ _MMIO(0x9888), 0x046c3422 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x06614000 },
+	{ _MMIO(0x9888), 0x0c620044 },
+	{ _MMIO(0x9888), 0x10620000 },
+	{ _MMIO(0x9888), 0x06620000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x126c4e80 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x0a633400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x026c3321 },
+	{ _MMIO(0x9888), 0x046c342f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c2000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x06604000 },
+	{ _MMIO(0x9888), 0x0c630044 },
+	{ _MMIO(0x9888), 0x10630000 },
+	{ _MMIO(0x9888), 0x06630000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c00aa },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102f3800 },
+	{ _MMIO(0x9888), 0x144d0500 },
+	{ _MMIO(0x9888), 0x120d03c0 },
+	{ _MMIO(0x9888), 0x140d03cf },
+	{ _MMIO(0x9888), 0x0c0f0004 },
+	{ _MMIO(0x9888), 0x0c4e4000 },
+	{ _MMIO(0x9888), 0x042f0480 },
+	{ _MMIO(0x9888), 0x082f0000 },
+	{ _MMIO(0x9888), 0x022f0000 },
+	{ _MMIO(0x9888), 0x0a4c0090 },
+	{ _MMIO(0x9888), 0x064d0027 },
+	{ _MMIO(0x9888), 0x004d0000 },
+	{ _MMIO(0x9888), 0x000d0d40 },
+	{ _MMIO(0x9888), 0x020d803f },
+	{ _MMIO(0x9888), 0x040d8023 },
+	{ _MMIO(0x9888), 0x100d0000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020f0010 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x0e0f0050 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41901400 },
+	{ _MMIO(0x9888), 0x43901485 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x14152c00 },
+	{ _MMIO(0x9888), 0x16150005 },
+	{ _MMIO(0x9888), 0x121600a0 },
+	{ _MMIO(0x9888), 0x14352c00 },
+	{ _MMIO(0x9888), 0x16350005 },
+	{ _MMIO(0x9888), 0x123600a0 },
+	{ _MMIO(0x9888), 0x14552c00 },
+	{ _MMIO(0x9888), 0x16550005 },
+	{ _MMIO(0x9888), 0x125600a0 },
+	{ _MMIO(0x9888), 0x062f6000 },
+	{ _MMIO(0x9888), 0x022f2000 },
+	{ _MMIO(0x9888), 0x0c4c0050 },
+	{ _MMIO(0x9888), 0x0a4c0010 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0350 },
+	{ _MMIO(0x9888), 0x0c0fb000 },
+	{ _MMIO(0x9888), 0x0e0f00da },
+	{ _MMIO(0x9888), 0x182c0028 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x022dc000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x0c138000 },
+	{ _MMIO(0x9888), 0x0e132000 },
+	{ _MMIO(0x9888), 0x0413c000 },
+	{ _MMIO(0x9888), 0x1c140018 },
+	{ _MMIO(0x9888), 0x0c157000 },
+	{ _MMIO(0x9888), 0x0e150078 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04162180 },
+	{ _MMIO(0x9888), 0x02160000 },
+	{ _MMIO(0x9888), 0x04174000 },
+	{ _MMIO(0x9888), 0x0233a000 },
+	{ _MMIO(0x9888), 0x04333000 },
+	{ _MMIO(0x9888), 0x14348000 },
+	{ _MMIO(0x9888), 0x16348000 },
+	{ _MMIO(0x9888), 0x02357870 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04360043 },
+	{ _MMIO(0x9888), 0x02360000 },
+	{ _MMIO(0x9888), 0x04371000 },
+	{ _MMIO(0x9888), 0x0e538000 },
+	{ _MMIO(0x9888), 0x00538000 },
+	{ _MMIO(0x9888), 0x06533000 },
+	{ _MMIO(0x9888), 0x1c540020 },
+	{ _MMIO(0x9888), 0x12548000 },
+	{ _MMIO(0x9888), 0x0e557000 },
+	{ _MMIO(0x9888), 0x00557800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06560043 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x06571000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900060 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900060 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x12120000 },
+	{ _MMIO(0x9888), 0x12320000 },
+	{ _MMIO(0x9888), 0x12520000 },
+	{ _MMIO(0x9888), 0x002f8000 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0015 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f03a0 },
+	{ _MMIO(0x9888), 0x0c0ff000 },
+	{ _MMIO(0x9888), 0x0e0f0095 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x02108000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x02118000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x02121880 },
+	{ _MMIO(0x9888), 0x041219b5 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x02134000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x0c308000 },
+	{ _MMIO(0x9888), 0x0e304000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x0c318000 },
+	{ _MMIO(0x9888), 0x0e314000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x0c321a80 },
+	{ _MMIO(0x9888), 0x0e320033 },
+	{ _MMIO(0x9888), 0x06320031 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x0c334000 },
+	{ _MMIO(0x9888), 0x0e331000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0e508000 },
+	{ _MMIO(0x9888), 0x00508000 },
+	{ _MMIO(0x9888), 0x02504000 },
+	{ _MMIO(0x9888), 0x0e518000 },
+	{ _MMIO(0x9888), 0x00518000 },
+	{ _MMIO(0x9888), 0x02514000 },
+	{ _MMIO(0x9888), 0x0e521880 },
+	{ _MMIO(0x9888), 0x00521a80 },
+	{ _MMIO(0x9888), 0x02520033 },
+	{ _MMIO(0x9888), 0x0e534000 },
+	{ _MMIO(0x9888), 0x00534000 },
+	{ _MMIO(0x9888), 0x02531000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900062 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x12124d60 },
+	{ _MMIO(0x9888), 0x12322e60 },
+	{ _MMIO(0x9888), 0x12524d60 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0014 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0fe000 },
+	{ _MMIO(0x9888), 0x0e0f0097 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x002d8000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x04121fb7 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x00308000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x00318000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x00321b80 },
+	{ _MMIO(0x9888), 0x0632003f },
+	{ _MMIO(0x9888), 0x00334000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0250c000 },
+	{ _MMIO(0x9888), 0x0251c000 },
+	{ _MMIO(0x9888), 0x02521fb7 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x02535000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900063 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+	{ _MMIO(0xe458), 0x00001000 },
+	{ _MMIO(0xe558), 0x00003002 },
+	{ _MMIO(0xe658), 0x00005004 },
+	{ _MMIO(0xe758), 0x00011010 },
+	{ _MMIO(0xe45c), 0x00050012 },
+	{ _MMIO(0xe55c), 0x00052051 },
+	{ _MMIO(0xe65c), 0x00000008 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x121203e0 },
+	{ _MMIO(0x9888), 0x123203e0 },
+	{ _MMIO(0x9888), 0x125203e0 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0e0f006c },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x042d8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06114000 },
+	{ _MMIO(0x9888), 0x06120033 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x04308000 },
+	{ _MMIO(0x9888), 0x04318000 },
+	{ _MMIO(0x9888), 0x04321980 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x04334000 },
+	{ _MMIO(0x9888), 0x04504000 },
+	{ _MMIO(0x9888), 0x04514000 },
+	{ _MMIO(0x9888), 0x04520033 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x04531000 },
+	{ _MMIO(0x9888), 0x1190e000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x43900c00 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00100030 },
+	{ _MMIO(0x2774), 0x0000fff9 },
+	{ _MMIO(0x2778), 0x00000002 },
+	{ _MMIO(0x277c), 0x0000fffc },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000fff3 },
+	{ _MMIO(0x2788), 0x00100180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000ff3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00008003 },
+};
+
+static const struct i915_oa_reg mux_config_vme_pipe[] = {
+	{ _MMIO(0x9888), 0x141a5800 },
+	{ _MMIO(0x9888), 0x161a00c0 },
+	{ _MMIO(0x9888), 0x12180240 },
+	{ _MMIO(0x9888), 0x14180002 },
+	{ _MMIO(0x9888), 0x143a5800 },
+	{ _MMIO(0x9888), 0x163a00c0 },
+	{ _MMIO(0x9888), 0x12380240 },
+	{ _MMIO(0x9888), 0x14380002 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x022f8000 },
+	{ _MMIO(0x9888), 0x042f3000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c1500 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f9500 },
+	{ _MMIO(0x9888), 0x100f002a },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x0a2dc000 },
+	{ _MMIO(0x9888), 0x0c2dc000 },
+	{ _MMIO(0x9888), 0x04193000 },
+	{ _MMIO(0x9888), 0x081a28c1 },
+	{ _MMIO(0x9888), 0x001a0000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x0613c000 },
+	{ _MMIO(0x9888), 0x0813f000 },
+	{ _MMIO(0x9888), 0x00172000 },
+	{ _MMIO(0x9888), 0x06178000 },
+	{ _MMIO(0x9888), 0x0817a000 },
+	{ _MMIO(0x9888), 0x00180037 },
+	{ _MMIO(0x9888), 0x06180940 },
+	{ _MMIO(0x9888), 0x08180000 },
+	{ _MMIO(0x9888), 0x02180000 },
+	{ _MMIO(0x9888), 0x04183000 },
+	{ _MMIO(0x9888), 0x06393000 },
+	{ _MMIO(0x9888), 0x0c3a28c1 },
+	{ _MMIO(0x9888), 0x003a0000 },
+	{ _MMIO(0x9888), 0x0a33f000 },
+	{ _MMIO(0x9888), 0x0c33f000 },
+	{ _MMIO(0x9888), 0x0a37a000 },
+	{ _MMIO(0x9888), 0x0c37a000 },
+	{ _MMIO(0x9888), 0x0a380977 },
+	{ _MMIO(0x9888), 0x08380000 },
+	{ _MMIO(0x9888), 0x04380000 },
+	{ _MMIO(0x9888), 0x06383000 },
+	{ _MMIO(0x9888), 0x119000ff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900800 },
+	{ _MMIO(0x9888), 0x47901000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900844 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
+			const struct i915_oa_reg **regs,
+			int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_vme_pipe;
+	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x11810000 },
+	{ _MMIO(0x9888), 0x07810016 },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930040 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_sklgt2(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_VME_PIPE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_vme_pipe_mux_config(dev_priv,
+						dev_priv->perf.oa.mux_regs,
+						dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_vme_pipe;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_vme_pipe);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_vme_pipe;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_vme_pipe);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "f519e481-24d2-4d42-87c9-3fdd12c00202",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "fe47b29d-ae51-423e-bff4-27d965a95b60",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "e0ad5ae0-84ba-4f29-a723-1906c12cb774",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "9bc436dd-6130-4add-affc-283eb6eaa864",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "2ea0da8f-3527-4669-9d9d-13099a7435bf",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "d97d16af-028b-4cd1-a672-6210cb5513dd",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "9fb22842-e708-43f7-9752-e0e41670c39e",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "5378e2a1-4248-4188-a4ae-da25a794c603",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "f42cdd6a-b000-42cb-870f-5eb423a7f514",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "b9bf2423-d88c-4a7b-a051-627611d00dcc",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "2414a93d-d84f-406e-99c0-472161194b40",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "53a45d2d-170b-4cf5-b7bb-585120c8e2f5",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "b4cff514-a91e-4798-a0b3-426ca13fc9c1",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "7821d13b-9b8b-4405-9618-78cd56b62cce",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "893f1a4d-919d-4388-8cb7-746d73ea7259",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "41a24047-7484-4ead-ae37-de907e5ff2b2",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
+}
+
+static struct device_attribute dev_attr_vme_pipe_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_vme_pipe_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_vme_pipe[] = {
+	&dev_attr_vme_pipe_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_vme_pipe = {
+	.name = "95910492-943f-44bd-9461-390240f243fd",
+	.attrs =  attrs_vme_pipe,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "1651949f-0ac0-4cb1-a06f-dafd74a407d1",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_sklgt2(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+		if (ret)
+			goto error_vme_pipe;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+error_vme_pipe:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_sklgt2(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.h b/drivers/gpu/drm/i915/i915_oa_sklgt2.h
new file mode 100644
index 000000000000..f4397baf3328
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_SKLGT2_H__
+#define __I915_OA_SKLGT2_H__
+
+extern int i915_oa_n_builtin_metric_sets_sklgt2;
+
+extern int i915_oa_select_metric_set_sklgt2(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_sklgt2(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_sklgt2(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.c b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
new file mode 100644
index 000000000000..7765e22dfa17
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
@@ -0,0 +1,3039 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_sklgt3.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_VME_PIPE,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_sklgt3 = 18;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9888), 0x166c01e0 },
+	{ _MMIO(0x9888), 0x12170280 },
+	{ _MMIO(0x9888), 0x12370280 },
+	{ _MMIO(0x9888), 0x16ec01e0 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x1a4e0380 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x0a1b4000 },
+	{ _MMIO(0x9888), 0x1c1c0001 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x042f1000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c8400 },
+	{ _MMIO(0x9888), 0x0c4c0002 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f6600 },
+	{ _MMIO(0x9888), 0x100f0001 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x162ca200 },
+	{ _MMIO(0x9888), 0x062d8000 },
+	{ _MMIO(0x9888), 0x082d8000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x08133000 },
+	{ _MMIO(0x9888), 0x00170020 },
+	{ _MMIO(0x9888), 0x08170021 },
+	{ _MMIO(0x9888), 0x10170000 },
+	{ _MMIO(0x9888), 0x0633c000 },
+	{ _MMIO(0x9888), 0x0833c000 },
+	{ _MMIO(0x9888), 0x06370800 },
+	{ _MMIO(0x9888), 0x08370840 },
+	{ _MMIO(0x9888), 0x10370000 },
+	{ _MMIO(0x9888), 0x1ace0200 },
+	{ _MMIO(0x9888), 0x0aec5300 },
+	{ _MMIO(0x9888), 0x10ec0000 },
+	{ _MMIO(0x9888), 0x1cec0000 },
+	{ _MMIO(0x9888), 0x0a9b8000 },
+	{ _MMIO(0x9888), 0x1c9c0002 },
+	{ _MMIO(0x9888), 0x0ccc0002 },
+	{ _MMIO(0x9888), 0x0a8d8000 },
+	{ _MMIO(0x9888), 0x108f0001 },
+	{ _MMIO(0x9888), 0x16ac8000 },
+	{ _MMIO(0x9888), 0x0d933031 },
+	{ _MMIO(0x9888), 0x0f933e3f },
+	{ _MMIO(0x9888), 0x01933d00 },
+	{ _MMIO(0x9888), 0x0393073c },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1d930000 },
+	{ _MMIO(0x9888), 0x19930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190003f },
+	{ _MMIO(0x9888), 0x51907710 },
+	{ _MMIO(0x9888), 0x419020a0 },
+	{ _MMIO(0x9888), 0x55901515 },
+	{ _MMIO(0x9888), 0x45900529 },
+	{ _MMIO(0x9888), 0x47901025 },
+	{ _MMIO(0x9888), 0x57907770 },
+	{ _MMIO(0x9888), 0x49902100 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900108 },
+	{ _MMIO(0x9888), 0x59900007 },
+	{ _MMIO(0x9888), 0x43902108 },
+	{ _MMIO(0x9888), 0x53907777 },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x1a4e0820 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x064f0900 },
+	{ _MMIO(0x9888), 0x084f0032 },
+	{ _MMIO(0x9888), 0x0a4f1891 },
+	{ _MMIO(0x9888), 0x0c4f0e00 },
+	{ _MMIO(0x9888), 0x0e4f003c },
+	{ _MMIO(0x9888), 0x004f0d80 },
+	{ _MMIO(0x9888), 0x024f003b },
+	{ _MMIO(0x9888), 0x006c0002 },
+	{ _MMIO(0x9888), 0x086c0100 },
+	{ _MMIO(0x9888), 0x0c6c000c },
+	{ _MMIO(0x9888), 0x0e6c0b00 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x081b8000 },
+	{ _MMIO(0x9888), 0x0c1b4000 },
+	{ _MMIO(0x9888), 0x0e1b8000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1c8000 },
+	{ _MMIO(0x9888), 0x1c1c0024 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5bc000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x1a5c6000 },
+	{ _MMIO(0x9888), 0x1c5c001b },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2000 },
+	{ _MMIO(0x9888), 0x0c4c0208 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020d2000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2cc000 },
+	{ _MMIO(0x9888), 0x162cfb00 },
+	{ _MMIO(0x9888), 0x182c00be },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x19900157 },
+	{ _MMIO(0x9888), 0x1b900158 },
+	{ _MMIO(0x9888), 0x1d900105 },
+	{ _MMIO(0x9888), 0x1f900103 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x11900fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900863 },
+	{ _MMIO(0x9888), 0x47900802 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900802 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900002 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900c62 },
+	{ _MMIO(0x9888), 0x53903333 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0c0e001f },
+	{ _MMIO(0x9888), 0x0a0f0000 },
+	{ _MMIO(0x9888), 0x10116800 },
+	{ _MMIO(0x9888), 0x178a03e0 },
+	{ _MMIO(0x9888), 0x11824c00 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x13840020 },
+	{ _MMIO(0x9888), 0x11850019 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x01870c40 },
+	{ _MMIO(0x9888), 0x17880000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x040d4000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020e5400 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x080f0040 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x0e0f0040 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06110012 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x01898000 },
+	{ _MMIO(0x9888), 0x0d890100 },
+	{ _MMIO(0x9888), 0x03898000 },
+	{ _MMIO(0x9888), 0x09808000 },
+	{ _MMIO(0x9888), 0x0b808000 },
+	{ _MMIO(0x9888), 0x0380c000 },
+	{ _MMIO(0x9888), 0x0f8a0075 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x118a8000 },
+	{ _MMIO(0x9888), 0x1b8a4000 },
+	{ _MMIO(0x9888), 0x138a8000 },
+	{ _MMIO(0x9888), 0x1d81a000 },
+	{ _MMIO(0x9888), 0x15818000 },
+	{ _MMIO(0x9888), 0x17818000 },
+	{ _MMIO(0x9888), 0x0b820030 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x0d824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x05824000 },
+	{ _MMIO(0x9888), 0x0d830003 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x03838000 },
+	{ _MMIO(0x9888), 0x07838000 },
+	{ _MMIO(0x9888), 0x0b840980 },
+	{ _MMIO(0x9888), 0x03844d80 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x09850080 },
+	{ _MMIO(0x9888), 0x03850003 },
+	{ _MMIO(0x9888), 0x01850000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x09870032 },
+	{ _MMIO(0x9888), 0x01888052 },
+	{ _MMIO(0x9888), 0x11880000 },
+	{ _MMIO(0x9888), 0x09884000 },
+	{ _MMIO(0x9888), 0x1b931001 },
+	{ _MMIO(0x9888), 0x1d930001 },
+	{ _MMIO(0x9888), 0x19934000 },
+	{ _MMIO(0x9888), 0x1b958000 },
+	{ _MMIO(0x9888), 0x1d950094 },
+	{ _MMIO(0x9888), 0x19958000 },
+	{ _MMIO(0x9888), 0x09e58000 },
+	{ _MMIO(0x9888), 0x0be58000 },
+	{ _MMIO(0x9888), 0x03e5c000 },
+	{ _MMIO(0x9888), 0x0592c000 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d924000 },
+	{ _MMIO(0x9888), 0x0f924000 },
+	{ _MMIO(0x9888), 0x11928000 },
+	{ _MMIO(0x9888), 0x1392c000 },
+	{ _MMIO(0x9888), 0x09924000 },
+	{ _MMIO(0x9888), 0x01985000 },
+	{ _MMIO(0x9888), 0x07988000 },
+	{ _MMIO(0x9888), 0x09981000 },
+	{ _MMIO(0x9888), 0x0b982000 },
+	{ _MMIO(0x9888), 0x0d982000 },
+	{ _MMIO(0x9888), 0x0f989000 },
+	{ _MMIO(0x9888), 0x05982000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1190c080 },
+	{ _MMIO(0x9888), 0x51901150 },
+	{ _MMIO(0x9888), 0x41901400 },
+	{ _MMIO(0x9888), 0x55905111 },
+	{ _MMIO(0x9888), 0x45901400 },
+	{ _MMIO(0x9888), 0x479004a5 },
+	{ _MMIO(0x9888), 0x57903455 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b9000a0 },
+	{ _MMIO(0x9888), 0x59900001 },
+	{ _MMIO(0x9888), 0x43900005 },
+	{ _MMIO(0x9888), 0x53900455 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900064 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900150 },
+	{ _MMIO(0x9888), 0x21900151 },
+	{ _MMIO(0x9888), 0x23900152 },
+	{ _MMIO(0x9888), 0x25900153 },
+	{ _MMIO(0x9888), 0x27900154 },
+	{ _MMIO(0x9888), 0x29900155 },
+	{ _MMIO(0x9888), 0x2b900156 },
+	{ _MMIO(0x9888), 0x2d900157 },
+	{ _MMIO(0x9888), 0x2f90015f },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900160 },
+	{ _MMIO(0x9888), 0x21900161 },
+	{ _MMIO(0x9888), 0x23900162 },
+	{ _MMIO(0x9888), 0x25900163 },
+	{ _MMIO(0x9888), 0x27900164 },
+	{ _MMIO(0x9888), 0x29900165 },
+	{ _MMIO(0x9888), 0x2b900166 },
+	{ _MMIO(0x9888), 0x2d900167 },
+	{ _MMIO(0x9888), 0x2f900150 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended[] = {
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x141c8160 },
+	{ _MMIO(0x9888), 0x161c8015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4eaaa0 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0e6c0b01 },
+	{ _MMIO(0x9888), 0x006c0200 },
+	{ _MMIO(0x9888), 0x026c000c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x001c0041 },
+	{ _MMIO(0x9888), 0x061c4200 },
+	{ _MMIO(0x9888), 0x081c4443 },
+	{ _MMIO(0x9888), 0x0a1c4645 },
+	{ _MMIO(0x9888), 0x0c1c7647 },
+	{ _MMIO(0x9888), 0x041c7357 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x101c0000 },
+	{ _MMIO(0x9888), 0x1a1c0000 },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4caa2a },
+	{ _MMIO(0x9888), 0x0c4c02aa },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5515 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x11907fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900802 },
+	{ _MMIO(0x9888), 0x47900842 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900842 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900800 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c0760 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4e8020 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1ce000 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2a00 },
+	{ _MMIO(0x9888), 0x0c4c0280 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f1500 },
+	{ _MMIO(0x9888), 0x100f0140 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x182c00a0 },
+	{ _MMIO(0x9888), 0x03933300 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190030f },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900063 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x53903333 },
+	{ _MMIO(0x9888), 0x43900840 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x106c0232 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x004f1880 },
+	{ _MMIO(0x9888), 0x024f08bb },
+	{ _MMIO(0x9888), 0x044f001b },
+	{ _MMIO(0x9888), 0x046c0100 },
+	{ _MMIO(0x9888), 0x066c000b },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x041b8000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025bc000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x165c8000 },
+	{ _MMIO(0x9888), 0x185c8000 },
+	{ _MMIO(0x9888), 0x0a4c00a0 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x062cc000 },
+	{ _MMIO(0x9888), 0x082cc000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x07830000 },
+	{ _MMIO(0x9888), 0x1d950080 },
+	{ _MMIO(0x9888), 0x13928000 },
+	{ _MMIO(0x9888), 0x0f988000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900005 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x126c7b40 },
+	{ _MMIO(0x9888), 0x166c0020 },
+	{ _MMIO(0x9888), 0x0a603444 },
+	{ _MMIO(0x9888), 0x0a613400 },
+	{ _MMIO(0x9888), 0x1a4ea800 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0800 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x1c1c003c },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x10600000 },
+	{ _MMIO(0x9888), 0x04600000 },
+	{ _MMIO(0x9888), 0x0c610044 },
+	{ _MMIO(0x9888), 0x10610000 },
+	{ _MMIO(0x9888), 0x06610000 },
+	{ _MMIO(0x9888), 0x0c4c02a8 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0154 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190ffc0 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900021 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900400 },
+	{ _MMIO(0x9888), 0x43900421 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x126c02e0 },
+	{ _MMIO(0x9888), 0x146c0001 },
+	{ _MMIO(0x9888), 0x0a623400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x026c3324 },
+	{ _MMIO(0x9888), 0x046c3422 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x06614000 },
+	{ _MMIO(0x9888), 0x0c620044 },
+	{ _MMIO(0x9888), 0x10620000 },
+	{ _MMIO(0x9888), 0x06620000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x126c4e80 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x0a633400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x026c3321 },
+	{ _MMIO(0x9888), 0x046c342f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c2000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x06604000 },
+	{ _MMIO(0x9888), 0x0c630044 },
+	{ _MMIO(0x9888), 0x10630000 },
+	{ _MMIO(0x9888), 0x06630000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c00aa },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102f3800 },
+	{ _MMIO(0x9888), 0x144d0500 },
+	{ _MMIO(0x9888), 0x120d03c0 },
+	{ _MMIO(0x9888), 0x140d03cf },
+	{ _MMIO(0x9888), 0x0c0f0004 },
+	{ _MMIO(0x9888), 0x0c4e4000 },
+	{ _MMIO(0x9888), 0x042f0480 },
+	{ _MMIO(0x9888), 0x082f0000 },
+	{ _MMIO(0x9888), 0x022f0000 },
+	{ _MMIO(0x9888), 0x0a4c0090 },
+	{ _MMIO(0x9888), 0x064d0027 },
+	{ _MMIO(0x9888), 0x004d0000 },
+	{ _MMIO(0x9888), 0x000d0d40 },
+	{ _MMIO(0x9888), 0x020d803f },
+	{ _MMIO(0x9888), 0x040d8023 },
+	{ _MMIO(0x9888), 0x100d0000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020f0010 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x0e0f0050 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41901400 },
+	{ _MMIO(0x9888), 0x43901485 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x14152c00 },
+	{ _MMIO(0x9888), 0x16150005 },
+	{ _MMIO(0x9888), 0x121600a0 },
+	{ _MMIO(0x9888), 0x14352c00 },
+	{ _MMIO(0x9888), 0x16350005 },
+	{ _MMIO(0x9888), 0x123600a0 },
+	{ _MMIO(0x9888), 0x14552c00 },
+	{ _MMIO(0x9888), 0x16550005 },
+	{ _MMIO(0x9888), 0x125600a0 },
+	{ _MMIO(0x9888), 0x062f6000 },
+	{ _MMIO(0x9888), 0x022f2000 },
+	{ _MMIO(0x9888), 0x0c4c0050 },
+	{ _MMIO(0x9888), 0x0a4c0010 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0350 },
+	{ _MMIO(0x9888), 0x0c0fb000 },
+	{ _MMIO(0x9888), 0x0e0f00da },
+	{ _MMIO(0x9888), 0x182c0028 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x022dc000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x0c138000 },
+	{ _MMIO(0x9888), 0x0e132000 },
+	{ _MMIO(0x9888), 0x0413c000 },
+	{ _MMIO(0x9888), 0x1c140018 },
+	{ _MMIO(0x9888), 0x0c157000 },
+	{ _MMIO(0x9888), 0x0e150078 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04162180 },
+	{ _MMIO(0x9888), 0x02160000 },
+	{ _MMIO(0x9888), 0x04174000 },
+	{ _MMIO(0x9888), 0x0233a000 },
+	{ _MMIO(0x9888), 0x04333000 },
+	{ _MMIO(0x9888), 0x14348000 },
+	{ _MMIO(0x9888), 0x16348000 },
+	{ _MMIO(0x9888), 0x02357870 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04360043 },
+	{ _MMIO(0x9888), 0x02360000 },
+	{ _MMIO(0x9888), 0x04371000 },
+	{ _MMIO(0x9888), 0x0e538000 },
+	{ _MMIO(0x9888), 0x00538000 },
+	{ _MMIO(0x9888), 0x06533000 },
+	{ _MMIO(0x9888), 0x1c540020 },
+	{ _MMIO(0x9888), 0x12548000 },
+	{ _MMIO(0x9888), 0x0e557000 },
+	{ _MMIO(0x9888), 0x00557800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06560043 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x06571000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900060 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900060 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x12120000 },
+	{ _MMIO(0x9888), 0x12320000 },
+	{ _MMIO(0x9888), 0x12520000 },
+	{ _MMIO(0x9888), 0x002f8000 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0015 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f03a0 },
+	{ _MMIO(0x9888), 0x0c0ff000 },
+	{ _MMIO(0x9888), 0x0e0f0095 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x02108000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x02118000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x02121880 },
+	{ _MMIO(0x9888), 0x041219b5 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x02134000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x0c308000 },
+	{ _MMIO(0x9888), 0x0e304000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x0c318000 },
+	{ _MMIO(0x9888), 0x0e314000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x0c321a80 },
+	{ _MMIO(0x9888), 0x0e320033 },
+	{ _MMIO(0x9888), 0x06320031 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x0c334000 },
+	{ _MMIO(0x9888), 0x0e331000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0e508000 },
+	{ _MMIO(0x9888), 0x00508000 },
+	{ _MMIO(0x9888), 0x02504000 },
+	{ _MMIO(0x9888), 0x0e518000 },
+	{ _MMIO(0x9888), 0x00518000 },
+	{ _MMIO(0x9888), 0x02514000 },
+	{ _MMIO(0x9888), 0x0e521880 },
+	{ _MMIO(0x9888), 0x00521a80 },
+	{ _MMIO(0x9888), 0x02520033 },
+	{ _MMIO(0x9888), 0x0e534000 },
+	{ _MMIO(0x9888), 0x00534000 },
+	{ _MMIO(0x9888), 0x02531000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900062 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x12124d60 },
+	{ _MMIO(0x9888), 0x12322e60 },
+	{ _MMIO(0x9888), 0x12524d60 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0014 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0fe000 },
+	{ _MMIO(0x9888), 0x0e0f0097 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x002d8000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x04121fb7 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x00308000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x00318000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x00321b80 },
+	{ _MMIO(0x9888), 0x0632003f },
+	{ _MMIO(0x9888), 0x00334000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0250c000 },
+	{ _MMIO(0x9888), 0x0251c000 },
+	{ _MMIO(0x9888), 0x02521fb7 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x02535000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900063 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x121203e0 },
+	{ _MMIO(0x9888), 0x123203e0 },
+	{ _MMIO(0x9888), 0x125203e0 },
+	{ _MMIO(0x9888), 0x129203e0 },
+	{ _MMIO(0x9888), 0x12b203e0 },
+	{ _MMIO(0x9888), 0x12d203e0 },
+	{ _MMIO(0x9888), 0x024ec000 },
+	{ _MMIO(0x9888), 0x044ec000 },
+	{ _MMIO(0x9888), 0x064ec000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c0042 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f006d },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x042d8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06114000 },
+	{ _MMIO(0x9888), 0x06120033 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x04308000 },
+	{ _MMIO(0x9888), 0x04318000 },
+	{ _MMIO(0x9888), 0x04321980 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x04334000 },
+	{ _MMIO(0x9888), 0x04504000 },
+	{ _MMIO(0x9888), 0x04514000 },
+	{ _MMIO(0x9888), 0x04520033 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x04531000 },
+	{ _MMIO(0x9888), 0x00af8000 },
+	{ _MMIO(0x9888), 0x0acc0001 },
+	{ _MMIO(0x9888), 0x008d8000 },
+	{ _MMIO(0x9888), 0x028da000 },
+	{ _MMIO(0x9888), 0x0c8fb000 },
+	{ _MMIO(0x9888), 0x0e8f0001 },
+	{ _MMIO(0x9888), 0x06ac8000 },
+	{ _MMIO(0x9888), 0x02ad4000 },
+	{ _MMIO(0x9888), 0x02908000 },
+	{ _MMIO(0x9888), 0x02918000 },
+	{ _MMIO(0x9888), 0x02921980 },
+	{ _MMIO(0x9888), 0x00920000 },
+	{ _MMIO(0x9888), 0x02934000 },
+	{ _MMIO(0x9888), 0x02b04000 },
+	{ _MMIO(0x9888), 0x02b14000 },
+	{ _MMIO(0x9888), 0x02b20033 },
+	{ _MMIO(0x9888), 0x00b20000 },
+	{ _MMIO(0x9888), 0x02b31000 },
+	{ _MMIO(0x9888), 0x00d08000 },
+	{ _MMIO(0x9888), 0x00d18000 },
+	{ _MMIO(0x9888), 0x00d21980 },
+	{ _MMIO(0x9888), 0x00d34000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900402 },
+	{ _MMIO(0x9888), 0x53901550 },
+	{ _MMIO(0x9888), 0x45900080 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00100030 },
+	{ _MMIO(0x2774), 0x0000fff9 },
+	{ _MMIO(0x2778), 0x00000002 },
+	{ _MMIO(0x277c), 0x0000fffc },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000fff3 },
+	{ _MMIO(0x2788), 0x00100180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000ff3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00008003 },
+};
+
+static const struct i915_oa_reg mux_config_vme_pipe[] = {
+	{ _MMIO(0x9888), 0x141a5800 },
+	{ _MMIO(0x9888), 0x161a00c0 },
+	{ _MMIO(0x9888), 0x12180240 },
+	{ _MMIO(0x9888), 0x14180002 },
+	{ _MMIO(0x9888), 0x149a5800 },
+	{ _MMIO(0x9888), 0x169a00c0 },
+	{ _MMIO(0x9888), 0x12980240 },
+	{ _MMIO(0x9888), 0x14980002 },
+	{ _MMIO(0x9888), 0x1a4e3fc0 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x022f8000 },
+	{ _MMIO(0x9888), 0x042f3000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c9500 },
+	{ _MMIO(0x9888), 0x0c4c002a },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0015 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c000a },
+	{ _MMIO(0x9888), 0x04193000 },
+	{ _MMIO(0x9888), 0x081a28c1 },
+	{ _MMIO(0x9888), 0x001a0000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x0613c000 },
+	{ _MMIO(0x9888), 0x0813f000 },
+	{ _MMIO(0x9888), 0x00172000 },
+	{ _MMIO(0x9888), 0x06178000 },
+	{ _MMIO(0x9888), 0x0817a000 },
+	{ _MMIO(0x9888), 0x00180037 },
+	{ _MMIO(0x9888), 0x06180940 },
+	{ _MMIO(0x9888), 0x08180000 },
+	{ _MMIO(0x9888), 0x02180000 },
+	{ _MMIO(0x9888), 0x04183000 },
+	{ _MMIO(0x9888), 0x04afc000 },
+	{ _MMIO(0x9888), 0x06af3000 },
+	{ _MMIO(0x9888), 0x0acc4000 },
+	{ _MMIO(0x9888), 0x0ccc0015 },
+	{ _MMIO(0x9888), 0x0a8da000 },
+	{ _MMIO(0x9888), 0x0c8da000 },
+	{ _MMIO(0x9888), 0x0e8f4000 },
+	{ _MMIO(0x9888), 0x108f0015 },
+	{ _MMIO(0x9888), 0x16aca000 },
+	{ _MMIO(0x9888), 0x18ac000a },
+	{ _MMIO(0x9888), 0x06993000 },
+	{ _MMIO(0x9888), 0x0c9a28c1 },
+	{ _MMIO(0x9888), 0x009a0000 },
+	{ _MMIO(0x9888), 0x0a93f000 },
+	{ _MMIO(0x9888), 0x0c93f000 },
+	{ _MMIO(0x9888), 0x0a97a000 },
+	{ _MMIO(0x9888), 0x0c97a000 },
+	{ _MMIO(0x9888), 0x0a980977 },
+	{ _MMIO(0x9888), 0x08980000 },
+	{ _MMIO(0x9888), 0x04980000 },
+	{ _MMIO(0x9888), 0x06983000 },
+	{ _MMIO(0x9888), 0x119000ff },
+	{ _MMIO(0x9888), 0x51900050 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x55900115 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x47900884 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900002 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
+			const struct i915_oa_reg **regs,
+			int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_vme_pipe;
+	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x11810000 },
+	{ _MMIO(0x9888), 0x07810013 },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930040 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_sklgt3(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_VME_PIPE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_vme_pipe_mux_config(dev_priv,
+						dev_priv->perf.oa.mux_regs,
+						dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_vme_pipe;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_vme_pipe);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_vme_pipe;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_vme_pipe);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "4616d450-2393-4836-8146-53c5ed84d359",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "4320492b-fd03-42ac-922f-dbe1ef3b7b58",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "bd2d9cae-b9ec-4f5b-9d2f-934bed398a2d",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "4ca0f3fe-7fd3-4924-98cb-1807d9879767",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "a0c0172c-ee13-403d-99ff-2bdf6936cf14",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "52435e0b-f188-42ea-8680-21a56ee20dee",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "27076eeb-49f3-4fed-8423-c66506005c63",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "8071b409-c39a-4674-94d7-32962ecfb512",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "5e0b391e-9ea8-4901-b2ff-b64ff616c7ed",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "25dc828e-1d2d-426e-9546-a1d4233cdf16",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "3dba9405-2d7e-4d70-8199-e734e82fd6bf",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "76935d7b-09c9-46bf-87f1-c18b4a86ebe5",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "1b34c0d6-4f4c-4d7b-833f-4aaf236d87a6",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "b375c985-9953-455b-bda2-b03f7594e9db",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "3e2be2bb-884a-49bb-82c5-2358e6bd5f2d",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "2d80a648-7b5a-4e92-bbe7-3b5c76f2e221",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
+}
+
+static struct device_attribute dev_attr_vme_pipe_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_vme_pipe_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_vme_pipe[] = {
+	&dev_attr_vme_pipe_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_vme_pipe = {
+	.name = "cfae9232-6ffc-42cc-a703-9790016925f0",
+	.attrs =  attrs_vme_pipe,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "2b985803-d3c9-4629-8a4f-634bfecba0e8",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_sklgt3(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+		if (ret)
+			goto error_vme_pipe;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+error_vme_pipe:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_sklgt3(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.h b/drivers/gpu/drm/i915/i915_oa_sklgt3.h
new file mode 100644
index 000000000000..c0accb1f9b74
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_SKLGT3_H__
+#define __I915_OA_SKLGT3_H__
+
+extern int i915_oa_n_builtin_metric_sets_sklgt3;
+
+extern int i915_oa_select_metric_set_sklgt3(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_sklgt3(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_sklgt3(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.c b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
new file mode 100644
index 000000000000..9ddab43a2176
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
@@ -0,0 +1,3093 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_sklgt4.h"
+
+enum metric_set_id {
+	METRIC_SET_ID_RENDER_BASIC = 1,
+	METRIC_SET_ID_COMPUTE_BASIC,
+	METRIC_SET_ID_RENDER_PIPE_PROFILE,
+	METRIC_SET_ID_MEMORY_READS,
+	METRIC_SET_ID_MEMORY_WRITES,
+	METRIC_SET_ID_COMPUTE_EXTENDED,
+	METRIC_SET_ID_COMPUTE_L3_CACHE,
+	METRIC_SET_ID_HDC_AND_SF,
+	METRIC_SET_ID_L3_1,
+	METRIC_SET_ID_L3_2,
+	METRIC_SET_ID_L3_3,
+	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
+	METRIC_SET_ID_SAMPLER,
+	METRIC_SET_ID_TDL_1,
+	METRIC_SET_ID_TDL_2,
+	METRIC_SET_ID_COMPUTE_EXTRA,
+	METRIC_SET_ID_VME_PIPE,
+	METRIC_SET_ID_TEST_OA,
+};
+
+int i915_oa_n_builtin_metric_sets_sklgt4 = 18;
+
+static const struct i915_oa_reg b_counter_config_render_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9888), 0x166c01e0 },
+	{ _MMIO(0x9888), 0x12170280 },
+	{ _MMIO(0x9888), 0x12370280 },
+	{ _MMIO(0x9888), 0x16ec01e0 },
+	{ _MMIO(0x9888), 0x176c01e0 },
+	{ _MMIO(0x9888), 0x11930317 },
+	{ _MMIO(0x9888), 0x159303df },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x1a4e03b0 },
+	{ _MMIO(0x9888), 0x0a6c0053 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x0a1b4000 },
+	{ _MMIO(0x9888), 0x1c1c0001 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x042f1000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4ca400 },
+	{ _MMIO(0x9888), 0x0c4c0002 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f5600 },
+	{ _MMIO(0x9888), 0x100f0001 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x062d8000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x08133000 },
+	{ _MMIO(0x9888), 0x00170020 },
+	{ _MMIO(0x9888), 0x08170021 },
+	{ _MMIO(0x9888), 0x10170000 },
+	{ _MMIO(0x9888), 0x0633c000 },
+	{ _MMIO(0x9888), 0x06370800 },
+	{ _MMIO(0x9888), 0x10370000 },
+	{ _MMIO(0x9888), 0x1ace0230 },
+	{ _MMIO(0x9888), 0x0aec5300 },
+	{ _MMIO(0x9888), 0x10ec0000 },
+	{ _MMIO(0x9888), 0x1cec0000 },
+	{ _MMIO(0x9888), 0x0a9b8000 },
+	{ _MMIO(0x9888), 0x1c9c0002 },
+	{ _MMIO(0x9888), 0x0acc2000 },
+	{ _MMIO(0x9888), 0x0ccc0002 },
+	{ _MMIO(0x9888), 0x088d8000 },
+	{ _MMIO(0x9888), 0x0a8d8000 },
+	{ _MMIO(0x9888), 0x0e8f1000 },
+	{ _MMIO(0x9888), 0x108f0001 },
+	{ _MMIO(0x9888), 0x16ac8800 },
+	{ _MMIO(0x9888), 0x1b4e0020 },
+	{ _MMIO(0x9888), 0x096c5300 },
+	{ _MMIO(0x9888), 0x116c0000 },
+	{ _MMIO(0x9888), 0x1d6c0000 },
+	{ _MMIO(0x9888), 0x091b8000 },
+	{ _MMIO(0x9888), 0x1b1c8000 },
+	{ _MMIO(0x9888), 0x0b4c2000 },
+	{ _MMIO(0x9888), 0x090d8000 },
+	{ _MMIO(0x9888), 0x0f0f1000 },
+	{ _MMIO(0x9888), 0x172c0800 },
+	{ _MMIO(0x9888), 0x0d933031 },
+	{ _MMIO(0x9888), 0x0f933e3f },
+	{ _MMIO(0x9888), 0x01933d00 },
+	{ _MMIO(0x9888), 0x0393073c },
+	{ _MMIO(0x9888), 0x0593000e },
+	{ _MMIO(0x9888), 0x1d930000 },
+	{ _MMIO(0x9888), 0x19930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x2b908000 },
+	{ _MMIO(0x9888), 0x2d908000 },
+	{ _MMIO(0x9888), 0x2f908000 },
+	{ _MMIO(0x9888), 0x31908000 },
+	{ _MMIO(0x9888), 0x15908000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190003f },
+	{ _MMIO(0x9888), 0x5190ff30 },
+	{ _MMIO(0x9888), 0x41900060 },
+	{ _MMIO(0x9888), 0x55903033 },
+	{ _MMIO(0x9888), 0x45901421 },
+	{ _MMIO(0x9888), 0x47900803 },
+	{ _MMIO(0x9888), 0x5790fff1 },
+	{ _MMIO(0x9888), 0x49900001 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x5990000f },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x5390ffff },
+};
+
+static int
+get_render_basic_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_basic;
+	lens[n] = ARRAY_SIZE(mux_config_render_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_basic[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_basic[] = {
+	{ _MMIO(0x9888), 0x104f00e0 },
+	{ _MMIO(0x9888), 0x124f1c00 },
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x1a4e0820 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x064f0900 },
+	{ _MMIO(0x9888), 0x084f0032 },
+	{ _MMIO(0x9888), 0x0a4f1891 },
+	{ _MMIO(0x9888), 0x0c4f0e00 },
+	{ _MMIO(0x9888), 0x0e4f003c },
+	{ _MMIO(0x9888), 0x004f0d80 },
+	{ _MMIO(0x9888), 0x024f003b },
+	{ _MMIO(0x9888), 0x006c0002 },
+	{ _MMIO(0x9888), 0x086c0100 },
+	{ _MMIO(0x9888), 0x0c6c000c },
+	{ _MMIO(0x9888), 0x0e6c0b00 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x081b8000 },
+	{ _MMIO(0x9888), 0x0c1b4000 },
+	{ _MMIO(0x9888), 0x0e1b8000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1c8000 },
+	{ _MMIO(0x9888), 0x1c1c0024 },
+	{ _MMIO(0x9888), 0x065b8000 },
+	{ _MMIO(0x9888), 0x085b4000 },
+	{ _MMIO(0x9888), 0x0a5bc000 },
+	{ _MMIO(0x9888), 0x0c5b8000 },
+	{ _MMIO(0x9888), 0x0e5b4000 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025b4000 },
+	{ _MMIO(0x9888), 0x1a5c6000 },
+	{ _MMIO(0x9888), 0x1c5c001b },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2000 },
+	{ _MMIO(0x9888), 0x0c4c0208 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020d2000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2cc000 },
+	{ _MMIO(0x9888), 0x162cfb00 },
+	{ _MMIO(0x9888), 0x182c00be },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x19900157 },
+	{ _MMIO(0x9888), 0x1b900158 },
+	{ _MMIO(0x9888), 0x1d900105 },
+	{ _MMIO(0x9888), 0x1f900103 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x11900fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900821 },
+	{ _MMIO(0x9888), 0x47900802 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900802 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900002 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900422 },
+	{ _MMIO(0x9888), 0x53905555 },
+};
+
+static int
+get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_basic;
+	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007ffea },
+	{ _MMIO(0x2774), 0x00007ffc },
+	{ _MMIO(0x2778), 0x0007affa },
+	{ _MMIO(0x277c), 0x0000f5fd },
+	{ _MMIO(0x2780), 0x00079ffa },
+	{ _MMIO(0x2784), 0x0000f3fb },
+	{ _MMIO(0x2788), 0x0007bf7a },
+	{ _MMIO(0x278c), 0x0000f7e7 },
+	{ _MMIO(0x2790), 0x0007fefa },
+	{ _MMIO(0x2794), 0x0000f7cf },
+	{ _MMIO(0x2798), 0x00077ffa },
+	{ _MMIO(0x279c), 0x0000efdf },
+	{ _MMIO(0x27a0), 0x0006fffa },
+	{ _MMIO(0x27a4), 0x0000cfbf },
+	{ _MMIO(0x27a8), 0x0003fffa },
+	{ _MMIO(0x27ac), 0x00005f7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
+	{ _MMIO(0x9888), 0x0c0e001f },
+	{ _MMIO(0x9888), 0x0a0f0000 },
+	{ _MMIO(0x9888), 0x10116800 },
+	{ _MMIO(0x9888), 0x178a03e0 },
+	{ _MMIO(0x9888), 0x11824c00 },
+	{ _MMIO(0x9888), 0x11830020 },
+	{ _MMIO(0x9888), 0x13840020 },
+	{ _MMIO(0x9888), 0x11850019 },
+	{ _MMIO(0x9888), 0x11860007 },
+	{ _MMIO(0x9888), 0x01870c40 },
+	{ _MMIO(0x9888), 0x17880000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0a4c0040 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x040d4000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020e5400 },
+	{ _MMIO(0x9888), 0x000e0000 },
+	{ _MMIO(0x9888), 0x080f0040 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x100f0000 },
+	{ _MMIO(0x9888), 0x0e0f0040 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06110012 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x01898000 },
+	{ _MMIO(0x9888), 0x0d890100 },
+	{ _MMIO(0x9888), 0x03898000 },
+	{ _MMIO(0x9888), 0x09808000 },
+	{ _MMIO(0x9888), 0x0b808000 },
+	{ _MMIO(0x9888), 0x0380c000 },
+	{ _MMIO(0x9888), 0x0f8a0075 },
+	{ _MMIO(0x9888), 0x1d8a0000 },
+	{ _MMIO(0x9888), 0x118a8000 },
+	{ _MMIO(0x9888), 0x1b8a4000 },
+	{ _MMIO(0x9888), 0x138a8000 },
+	{ _MMIO(0x9888), 0x1d81a000 },
+	{ _MMIO(0x9888), 0x15818000 },
+	{ _MMIO(0x9888), 0x17818000 },
+	{ _MMIO(0x9888), 0x0b820030 },
+	{ _MMIO(0x9888), 0x07828000 },
+	{ _MMIO(0x9888), 0x0d824000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x05824000 },
+	{ _MMIO(0x9888), 0x0d830003 },
+	{ _MMIO(0x9888), 0x0583000c },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x03838000 },
+	{ _MMIO(0x9888), 0x07838000 },
+	{ _MMIO(0x9888), 0x0b840980 },
+	{ _MMIO(0x9888), 0x03844d80 },
+	{ _MMIO(0x9888), 0x11840000 },
+	{ _MMIO(0x9888), 0x09848000 },
+	{ _MMIO(0x9888), 0x09850080 },
+	{ _MMIO(0x9888), 0x03850003 },
+	{ _MMIO(0x9888), 0x01850000 },
+	{ _MMIO(0x9888), 0x07860000 },
+	{ _MMIO(0x9888), 0x0f860400 },
+	{ _MMIO(0x9888), 0x09870032 },
+	{ _MMIO(0x9888), 0x01888052 },
+	{ _MMIO(0x9888), 0x11880000 },
+	{ _MMIO(0x9888), 0x09884000 },
+	{ _MMIO(0x9888), 0x1b931001 },
+	{ _MMIO(0x9888), 0x1d930001 },
+	{ _MMIO(0x9888), 0x19934000 },
+	{ _MMIO(0x9888), 0x1b958000 },
+	{ _MMIO(0x9888), 0x1d950094 },
+	{ _MMIO(0x9888), 0x19958000 },
+	{ _MMIO(0x9888), 0x09e58000 },
+	{ _MMIO(0x9888), 0x0be58000 },
+	{ _MMIO(0x9888), 0x03e5c000 },
+	{ _MMIO(0x9888), 0x0592c000 },
+	{ _MMIO(0x9888), 0x0b928000 },
+	{ _MMIO(0x9888), 0x0d924000 },
+	{ _MMIO(0x9888), 0x0f924000 },
+	{ _MMIO(0x9888), 0x11928000 },
+	{ _MMIO(0x9888), 0x1392c000 },
+	{ _MMIO(0x9888), 0x09924000 },
+	{ _MMIO(0x9888), 0x01985000 },
+	{ _MMIO(0x9888), 0x07988000 },
+	{ _MMIO(0x9888), 0x09981000 },
+	{ _MMIO(0x9888), 0x0b982000 },
+	{ _MMIO(0x9888), 0x0d982000 },
+	{ _MMIO(0x9888), 0x0f989000 },
+	{ _MMIO(0x9888), 0x05982000 },
+	{ _MMIO(0x9888), 0x13904000 },
+	{ _MMIO(0x9888), 0x21904000 },
+	{ _MMIO(0x9888), 0x23904000 },
+	{ _MMIO(0x9888), 0x25908000 },
+	{ _MMIO(0x9888), 0x27904000 },
+	{ _MMIO(0x9888), 0x29908000 },
+	{ _MMIO(0x9888), 0x2b904000 },
+	{ _MMIO(0x9888), 0x2f904000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x15904000 },
+	{ _MMIO(0x9888), 0x17908000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b904000 },
+	{ _MMIO(0x9888), 0x1190c080 },
+	{ _MMIO(0x9888), 0x51901110 },
+	{ _MMIO(0x9888), 0x41900440 },
+	{ _MMIO(0x9888), 0x55901111 },
+	{ _MMIO(0x9888), 0x45900400 },
+	{ _MMIO(0x9888), 0x47900c21 },
+	{ _MMIO(0x9888), 0x57901411 },
+	{ _MMIO(0x9888), 0x49900042 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900024 },
+	{ _MMIO(0x9888), 0x59900001 },
+	{ _MMIO(0x9888), 0x43900841 },
+	{ _MMIO(0x9888), 0x53900411 },
+};
+
+static int
+get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
+				   const struct i915_oa_reg **regs,
+				   int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_render_pipe_profile;
+	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_reads[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f872 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_reads[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f900064 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900150 },
+	{ _MMIO(0x9888), 0x21900151 },
+	{ _MMIO(0x9888), 0x23900152 },
+	{ _MMIO(0x9888), 0x25900153 },
+	{ _MMIO(0x9888), 0x27900154 },
+	{ _MMIO(0x9888), 0x29900155 },
+	{ _MMIO(0x9888), 0x2b900156 },
+	{ _MMIO(0x9888), 0x2d900157 },
+	{ _MMIO(0x9888), 0x2f90015f },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
+			    const struct i915_oa_reg **regs,
+			    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_reads;
+	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_memory_writes[] = {
+	{ _MMIO(0x272c), 0xffffffff },
+	{ _MMIO(0x2728), 0xffffffff },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x271c), 0xffffffff },
+	{ _MMIO(0x2718), 0xffffffff },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x274c), 0x86543210 },
+	{ _MMIO(0x2748), 0x86543210 },
+	{ _MMIO(0x2744), 0x00006667 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x275c), 0x86543210 },
+	{ _MMIO(0x2758), 0x86543210 },
+	{ _MMIO(0x2754), 0x00006465 },
+	{ _MMIO(0x2750), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007f81a },
+	{ _MMIO(0x2774), 0x0000fe00 },
+	{ _MMIO(0x2778), 0x0007f82a },
+	{ _MMIO(0x277c), 0x0000fe00 },
+	{ _MMIO(0x2780), 0x0007f822 },
+	{ _MMIO(0x2784), 0x0000fe00 },
+	{ _MMIO(0x2788), 0x0007f8ba },
+	{ _MMIO(0x278c), 0x0000fe00 },
+	{ _MMIO(0x2790), 0x0007f87a },
+	{ _MMIO(0x2794), 0x0000fe00 },
+	{ _MMIO(0x2798), 0x0007f8ea },
+	{ _MMIO(0x279c), 0x0000fe00 },
+	{ _MMIO(0x27a0), 0x0007f8e2 },
+	{ _MMIO(0x27a4), 0x0000fe00 },
+	{ _MMIO(0x27a8), 0x0007f8f2 },
+	{ _MMIO(0x27ac), 0x0000fe00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00015014 },
+	{ _MMIO(0xe658), 0x00025024 },
+	{ _MMIO(0xe758), 0x00035034 },
+	{ _MMIO(0xe45c), 0x00045044 },
+	{ _MMIO(0xe55c), 0x00055054 },
+	{ _MMIO(0xe65c), 0x00065064 },
+};
+
+static const struct i915_oa_reg mux_config_memory_writes[] = {
+	{ _MMIO(0x9888), 0x11810c00 },
+	{ _MMIO(0x9888), 0x1381001a },
+	{ _MMIO(0x9888), 0x37906800 },
+	{ _MMIO(0x9888), 0x3f901000 },
+	{ _MMIO(0x9888), 0x03811300 },
+	{ _MMIO(0x9888), 0x05811b12 },
+	{ _MMIO(0x9888), 0x0781001a },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x17810000 },
+	{ _MMIO(0x9888), 0x19810000 },
+	{ _MMIO(0x9888), 0x1b810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930055 },
+	{ _MMIO(0x9888), 0x03e58000 },
+	{ _MMIO(0x9888), 0x05e5c000 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x13900160 },
+	{ _MMIO(0x9888), 0x21900161 },
+	{ _MMIO(0x9888), 0x23900162 },
+	{ _MMIO(0x9888), 0x25900163 },
+	{ _MMIO(0x9888), 0x27900164 },
+	{ _MMIO(0x9888), 0x29900165 },
+	{ _MMIO(0x9888), 0x2b900166 },
+	{ _MMIO(0x9888), 0x2d900167 },
+	{ _MMIO(0x9888), 0x2f900150 },
+	{ _MMIO(0x9888), 0x31900105 },
+	{ _MMIO(0x9888), 0x15900103 },
+	{ _MMIO(0x9888), 0x17900101 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1d908000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c60 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900c63 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c63 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900063 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_memory_writes;
+	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extended[] = {
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fc2a },
+	{ _MMIO(0x2774), 0x0000bf00 },
+	{ _MMIO(0x2778), 0x0007fc6a },
+	{ _MMIO(0x277c), 0x0000bf00 },
+	{ _MMIO(0x2780), 0x0007fc92 },
+	{ _MMIO(0x2784), 0x0000bf00 },
+	{ _MMIO(0x2788), 0x0007fca2 },
+	{ _MMIO(0x278c), 0x0000bf00 },
+	{ _MMIO(0x2790), 0x0007fc32 },
+	{ _MMIO(0x2794), 0x0000bf00 },
+	{ _MMIO(0x2798), 0x0007fc9a },
+	{ _MMIO(0x279c), 0x0000bf00 },
+	{ _MMIO(0x27a0), 0x0007fe6a },
+	{ _MMIO(0x27a4), 0x0000bf00 },
+	{ _MMIO(0x27a8), 0x0007fe7a },
+	{ _MMIO(0x27ac), 0x0000bf00 },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00778008 },
+	{ _MMIO(0xe45c), 0x00088078 },
+	{ _MMIO(0xe55c), 0x00808708 },
+	{ _MMIO(0xe65c), 0x00a08908 },
+};
+
+static const struct i915_oa_reg mux_config_compute_extended[] = {
+	{ _MMIO(0x9888), 0x106c00e0 },
+	{ _MMIO(0x9888), 0x141c8160 },
+	{ _MMIO(0x9888), 0x161c8015 },
+	{ _MMIO(0x9888), 0x181c0120 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4eaaa0 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0e6c0b01 },
+	{ _MMIO(0x9888), 0x006c0200 },
+	{ _MMIO(0x9888), 0x026c000c },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x001c0041 },
+	{ _MMIO(0x9888), 0x061c4200 },
+	{ _MMIO(0x9888), 0x081c4443 },
+	{ _MMIO(0x9888), 0x0a1c4645 },
+	{ _MMIO(0x9888), 0x0c1c7647 },
+	{ _MMIO(0x9888), 0x041c7357 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x101c0000 },
+	{ _MMIO(0x9888), 0x1a1c0000 },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4caa2a },
+	{ _MMIO(0x9888), 0x0c4c02aa },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x000da000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x0c0f5400 },
+	{ _MMIO(0x9888), 0x0e0f5515 },
+	{ _MMIO(0x9888), 0x100f0155 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x11907fff },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900040 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900802 },
+	{ _MMIO(0x9888), 0x47900842 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900842 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x43900800 },
+	{ _MMIO(0x9888), 0x53900000 },
+};
+
+static int
+get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extended;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2770), 0x0007fffa },
+	{ _MMIO(0x2774), 0x0000fefe },
+	{ _MMIO(0x2778), 0x0007fffa },
+	{ _MMIO(0x277c), 0x0000fefd },
+	{ _MMIO(0x2790), 0x0007fffa },
+	{ _MMIO(0x2794), 0x0000fbef },
+	{ _MMIO(0x2798), 0x0007fffa },
+	{ _MMIO(0x279c), 0x0000fbdf },
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00000003 },
+	{ _MMIO(0xe658), 0x00002001 },
+	{ _MMIO(0xe758), 0x00101100 },
+	{ _MMIO(0xe45c), 0x00201200 },
+	{ _MMIO(0xe55c), 0x00301300 },
+	{ _MMIO(0xe65c), 0x00401400 },
+};
+
+static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
+	{ _MMIO(0x9888), 0x166c0760 },
+	{ _MMIO(0x9888), 0x1593001e },
+	{ _MMIO(0x9888), 0x3f900003 },
+	{ _MMIO(0x9888), 0x004e8000 },
+	{ _MMIO(0x9888), 0x0e4e8000 },
+	{ _MMIO(0x9888), 0x184e8000 },
+	{ _MMIO(0x9888), 0x1a4e8020 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x006c0051 },
+	{ _MMIO(0x9888), 0x066c5000 },
+	{ _MMIO(0x9888), 0x086c5c5d },
+	{ _MMIO(0x9888), 0x0e6c5e5f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x186c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x001b4000 },
+	{ _MMIO(0x9888), 0x061b8000 },
+	{ _MMIO(0x9888), 0x081bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x101c8000 },
+	{ _MMIO(0x9888), 0x1a1ce000 },
+	{ _MMIO(0x9888), 0x1c1c0030 },
+	{ _MMIO(0x9888), 0x004c8000 },
+	{ _MMIO(0x9888), 0x0a4c2a00 },
+	{ _MMIO(0x9888), 0x0c4c0280 },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f1500 },
+	{ _MMIO(0x9888), 0x100f0140 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162c0a00 },
+	{ _MMIO(0x9888), 0x182c00a0 },
+	{ _MMIO(0x9888), 0x03933300 },
+	{ _MMIO(0x9888), 0x05930032 },
+	{ _MMIO(0x9888), 0x11930000 },
+	{ _MMIO(0x9888), 0x1b930000 },
+	{ _MMIO(0x9888), 0x1d900157 },
+	{ _MMIO(0x9888), 0x1f900158 },
+	{ _MMIO(0x9888), 0x35900000 },
+	{ _MMIO(0x9888), 0x19908000 },
+	{ _MMIO(0x9888), 0x1b908000 },
+	{ _MMIO(0x9888), 0x1190030f },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900000 },
+	{ _MMIO(0x9888), 0x55900000 },
+	{ _MMIO(0x9888), 0x45900021 },
+	{ _MMIO(0x9888), 0x47900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x4b900000 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x53905555 },
+	{ _MMIO(0x9888), 0x43900000 },
+};
+
+static int
+get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
+				const struct i915_oa_reg **regs,
+				int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_l3_cache;
+	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x10800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000fdff },
+};
+
+static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
+	{ _MMIO(0x9888), 0x104f0232 },
+	{ _MMIO(0x9888), 0x124f4640 },
+	{ _MMIO(0x9888), 0x106c0232 },
+	{ _MMIO(0x9888), 0x11834400 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x004f1880 },
+	{ _MMIO(0x9888), 0x024f08bb },
+	{ _MMIO(0x9888), 0x044f001b },
+	{ _MMIO(0x9888), 0x046c0100 },
+	{ _MMIO(0x9888), 0x066c000b },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x041b8000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x005b8000 },
+	{ _MMIO(0x9888), 0x025bc000 },
+	{ _MMIO(0x9888), 0x045b4000 },
+	{ _MMIO(0x9888), 0x125c8000 },
+	{ _MMIO(0x9888), 0x145c8000 },
+	{ _MMIO(0x9888), 0x165c8000 },
+	{ _MMIO(0x9888), 0x185c8000 },
+	{ _MMIO(0x9888), 0x0a4c00a0 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x022cc000 },
+	{ _MMIO(0x9888), 0x042cc000 },
+	{ _MMIO(0x9888), 0x062cc000 },
+	{ _MMIO(0x9888), 0x082cc000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x0f828000 },
+	{ _MMIO(0x9888), 0x0f8305c0 },
+	{ _MMIO(0x9888), 0x09830000 },
+	{ _MMIO(0x9888), 0x07830000 },
+	{ _MMIO(0x9888), 0x1d950080 },
+	{ _MMIO(0x9888), 0x13928000 },
+	{ _MMIO(0x9888), 0x0f988000 },
+	{ _MMIO(0x9888), 0x31904000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x59900001 },
+	{ _MMIO(0x9888), 0x4b900040 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
+			  const struct i915_oa_reg **regs,
+			  int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_hdc_and_sf;
+	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00014002 },
+	{ _MMIO(0x277c), 0x0000c3ff },
+	{ _MMIO(0x2780), 0x00010002 },
+	{ _MMIO(0x2784), 0x0000c7ff },
+	{ _MMIO(0x2788), 0x00004002 },
+	{ _MMIO(0x278c), 0x0000d3ff },
+	{ _MMIO(0x2790), 0x00100700 },
+	{ _MMIO(0x2794), 0x0000ff1f },
+	{ _MMIO(0x2798), 0x00001402 },
+	{ _MMIO(0x279c), 0x0000fc3f },
+	{ _MMIO(0x27a0), 0x00001002 },
+	{ _MMIO(0x27a4), 0x0000fc7f },
+	{ _MMIO(0x27a8), 0x00000402 },
+	{ _MMIO(0x27ac), 0x0000fd3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_1[] = {
+	{ _MMIO(0x9888), 0x126c7b40 },
+	{ _MMIO(0x9888), 0x166c0020 },
+	{ _MMIO(0x9888), 0x0a603444 },
+	{ _MMIO(0x9888), 0x0a613400 },
+	{ _MMIO(0x9888), 0x1a4ea800 },
+	{ _MMIO(0x9888), 0x1c4e0002 },
+	{ _MMIO(0x9888), 0x024e8000 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x0c6c5327 },
+	{ _MMIO(0x9888), 0x0e6c5425 },
+	{ _MMIO(0x9888), 0x006c2a00 },
+	{ _MMIO(0x9888), 0x026c285b },
+	{ _MMIO(0x9888), 0x046c005c },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1c6c0000 },
+	{ _MMIO(0x9888), 0x1e6c0000 },
+	{ _MMIO(0x9888), 0x1a6c0800 },
+	{ _MMIO(0x9888), 0x0c1bc000 },
+	{ _MMIO(0x9888), 0x0e1bc000 },
+	{ _MMIO(0x9888), 0x001b8000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x1c1c003c },
+	{ _MMIO(0x9888), 0x121c8000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x10600000 },
+	{ _MMIO(0x9888), 0x04600000 },
+	{ _MMIO(0x9888), 0x0c610044 },
+	{ _MMIO(0x9888), 0x10610000 },
+	{ _MMIO(0x9888), 0x06610000 },
+	{ _MMIO(0x9888), 0x0c4c02a8 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0154 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x182c00aa },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190ffc0 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900420 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900021 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900400 },
+	{ _MMIO(0x9888), 0x43900421 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_l3_1_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_1;
+	lens[n] = ARRAY_SIZE(mux_config_l3_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_2[] = {
+	{ _MMIO(0x9888), 0x126c02e0 },
+	{ _MMIO(0x9888), 0x146c0001 },
+	{ _MMIO(0x9888), 0x0a623400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x064f4000 },
+	{ _MMIO(0x9888), 0x026c3324 },
+	{ _MMIO(0x9888), 0x046c3422 },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c0000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c0800 },
+	{ _MMIO(0x9888), 0x065b4000 },
+	{ _MMIO(0x9888), 0x1a5c1000 },
+	{ _MMIO(0x9888), 0x06614000 },
+	{ _MMIO(0x9888), 0x0c620044 },
+	{ _MMIO(0x9888), 0x10620000 },
+	{ _MMIO(0x9888), 0x06620000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c002a },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2cc000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_2_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_2;
+	lens[n] = ARRAY_SIZE(mux_config_l3_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_l3_3[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00100070 },
+	{ _MMIO(0x2774), 0x0000fff1 },
+	{ _MMIO(0x2778), 0x00028002 },
+	{ _MMIO(0x277c), 0x000087ff },
+	{ _MMIO(0x2780), 0x00020002 },
+	{ _MMIO(0x2784), 0x00008fff },
+	{ _MMIO(0x2788), 0x00008002 },
+	{ _MMIO(0x278c), 0x0000a7ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_l3_3[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_l3_3[] = {
+	{ _MMIO(0x9888), 0x126c4e80 },
+	{ _MMIO(0x9888), 0x146c0000 },
+	{ _MMIO(0x9888), 0x0a633400 },
+	{ _MMIO(0x9888), 0x044e8000 },
+	{ _MMIO(0x9888), 0x064e8000 },
+	{ _MMIO(0x9888), 0x084e8000 },
+	{ _MMIO(0x9888), 0x0a4e8000 },
+	{ _MMIO(0x9888), 0x0c4e8000 },
+	{ _MMIO(0x9888), 0x026c3321 },
+	{ _MMIO(0x9888), 0x046c342f },
+	{ _MMIO(0x9888), 0x106c0000 },
+	{ _MMIO(0x9888), 0x1a6c2000 },
+	{ _MMIO(0x9888), 0x021bc000 },
+	{ _MMIO(0x9888), 0x041bc000 },
+	{ _MMIO(0x9888), 0x061b4000 },
+	{ _MMIO(0x9888), 0x141c8000 },
+	{ _MMIO(0x9888), 0x161c8000 },
+	{ _MMIO(0x9888), 0x181c8000 },
+	{ _MMIO(0x9888), 0x1a1c1800 },
+	{ _MMIO(0x9888), 0x06604000 },
+	{ _MMIO(0x9888), 0x0c630044 },
+	{ _MMIO(0x9888), 0x10630000 },
+	{ _MMIO(0x9888), 0x06630000 },
+	{ _MMIO(0x9888), 0x084c8000 },
+	{ _MMIO(0x9888), 0x0a4c00aa },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0f4000 },
+	{ _MMIO(0x9888), 0x0e0f0055 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190f800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900002 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_l3_3_mux_config(struct drm_i915_private *dev_priv,
+		    const struct i915_oa_reg **regs,
+		    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_l3_3;
+	lens[n] = ARRAY_SIZE(mux_config_l3_3);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x30800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x0000efff },
+	{ _MMIO(0x2778), 0x00006000 },
+	{ _MMIO(0x277c), 0x0000f3ff },
+};
+
+static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
+	{ _MMIO(0x9888), 0x102f3800 },
+	{ _MMIO(0x9888), 0x144d0500 },
+	{ _MMIO(0x9888), 0x120d03c0 },
+	{ _MMIO(0x9888), 0x140d03cf },
+	{ _MMIO(0x9888), 0x0c0f0004 },
+	{ _MMIO(0x9888), 0x0c4e4000 },
+	{ _MMIO(0x9888), 0x042f0480 },
+	{ _MMIO(0x9888), 0x082f0000 },
+	{ _MMIO(0x9888), 0x022f0000 },
+	{ _MMIO(0x9888), 0x0a4c0090 },
+	{ _MMIO(0x9888), 0x064d0027 },
+	{ _MMIO(0x9888), 0x004d0000 },
+	{ _MMIO(0x9888), 0x000d0d40 },
+	{ _MMIO(0x9888), 0x020d803f },
+	{ _MMIO(0x9888), 0x040d8023 },
+	{ _MMIO(0x9888), 0x100d0000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x020f0010 },
+	{ _MMIO(0x9888), 0x000f0000 },
+	{ _MMIO(0x9888), 0x0e0f0050 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41901400 },
+	{ _MMIO(0x9888), 0x43901485 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900001 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
+					    const struct i915_oa_reg **regs,
+					    int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_rasterizer_and_pixel_backend;
+	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_sampler[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x70800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+	{ _MMIO(0x2770), 0x0000c000 },
+	{ _MMIO(0x2774), 0x0000e7ff },
+	{ _MMIO(0x2778), 0x00003000 },
+	{ _MMIO(0x277c), 0x0000f9ff },
+	{ _MMIO(0x2780), 0x00000c00 },
+	{ _MMIO(0x2784), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_sampler[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_sampler[] = {
+	{ _MMIO(0x9888), 0x14152c00 },
+	{ _MMIO(0x9888), 0x16150005 },
+	{ _MMIO(0x9888), 0x121600a0 },
+	{ _MMIO(0x9888), 0x14352c00 },
+	{ _MMIO(0x9888), 0x16350005 },
+	{ _MMIO(0x9888), 0x123600a0 },
+	{ _MMIO(0x9888), 0x14552c00 },
+	{ _MMIO(0x9888), 0x16550005 },
+	{ _MMIO(0x9888), 0x125600a0 },
+	{ _MMIO(0x9888), 0x062f6000 },
+	{ _MMIO(0x9888), 0x022f2000 },
+	{ _MMIO(0x9888), 0x0c4c0050 },
+	{ _MMIO(0x9888), 0x0a4c0010 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0350 },
+	{ _MMIO(0x9888), 0x0c0fb000 },
+	{ _MMIO(0x9888), 0x0e0f00da },
+	{ _MMIO(0x9888), 0x182c0028 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x022dc000 },
+	{ _MMIO(0x9888), 0x042d4000 },
+	{ _MMIO(0x9888), 0x0c138000 },
+	{ _MMIO(0x9888), 0x0e132000 },
+	{ _MMIO(0x9888), 0x0413c000 },
+	{ _MMIO(0x9888), 0x1c140018 },
+	{ _MMIO(0x9888), 0x0c157000 },
+	{ _MMIO(0x9888), 0x0e150078 },
+	{ _MMIO(0x9888), 0x10150000 },
+	{ _MMIO(0x9888), 0x04162180 },
+	{ _MMIO(0x9888), 0x02160000 },
+	{ _MMIO(0x9888), 0x04174000 },
+	{ _MMIO(0x9888), 0x0233a000 },
+	{ _MMIO(0x9888), 0x04333000 },
+	{ _MMIO(0x9888), 0x14348000 },
+	{ _MMIO(0x9888), 0x16348000 },
+	{ _MMIO(0x9888), 0x02357870 },
+	{ _MMIO(0x9888), 0x10350000 },
+	{ _MMIO(0x9888), 0x04360043 },
+	{ _MMIO(0x9888), 0x02360000 },
+	{ _MMIO(0x9888), 0x04371000 },
+	{ _MMIO(0x9888), 0x0e538000 },
+	{ _MMIO(0x9888), 0x00538000 },
+	{ _MMIO(0x9888), 0x06533000 },
+	{ _MMIO(0x9888), 0x1c540020 },
+	{ _MMIO(0x9888), 0x12548000 },
+	{ _MMIO(0x9888), 0x0e557000 },
+	{ _MMIO(0x9888), 0x00557800 },
+	{ _MMIO(0x9888), 0x10550000 },
+	{ _MMIO(0x9888), 0x06560043 },
+	{ _MMIO(0x9888), 0x02560000 },
+	{ _MMIO(0x9888), 0x06571000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900060 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900842 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900060 },
+};
+
+static int
+get_sampler_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_sampler;
+	lens[n] = ARRAY_SIZE(mux_config_sampler);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_1[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00000002 },
+	{ _MMIO(0x2774), 0x00007fff },
+	{ _MMIO(0x2778), 0x00000000 },
+	{ _MMIO(0x277c), 0x00009fff },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000efff },
+	{ _MMIO(0x2788), 0x00000000 },
+	{ _MMIO(0x278c), 0x0000f3ff },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000fdff },
+	{ _MMIO(0x2798), 0x00000000 },
+	{ _MMIO(0x279c), 0x0000fe7f },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_1[] = {
+	{ _MMIO(0x9888), 0x12120000 },
+	{ _MMIO(0x9888), 0x12320000 },
+	{ _MMIO(0x9888), 0x12520000 },
+	{ _MMIO(0x9888), 0x002f8000 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0015 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f03a0 },
+	{ _MMIO(0x9888), 0x0c0ff000 },
+	{ _MMIO(0x9888), 0x0e0f0095 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x0c2d8000 },
+	{ _MMIO(0x9888), 0x0e2d4000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x02108000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x02118000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x02121880 },
+	{ _MMIO(0x9888), 0x041219b5 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x02134000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x0c308000 },
+	{ _MMIO(0x9888), 0x0e304000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x0c318000 },
+	{ _MMIO(0x9888), 0x0e314000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x0c321a80 },
+	{ _MMIO(0x9888), 0x0e320033 },
+	{ _MMIO(0x9888), 0x06320031 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x0c334000 },
+	{ _MMIO(0x9888), 0x0e331000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0e508000 },
+	{ _MMIO(0x9888), 0x00508000 },
+	{ _MMIO(0x9888), 0x02504000 },
+	{ _MMIO(0x9888), 0x0e518000 },
+	{ _MMIO(0x9888), 0x00518000 },
+	{ _MMIO(0x9888), 0x02514000 },
+	{ _MMIO(0x9888), 0x0e521880 },
+	{ _MMIO(0x9888), 0x00521a80 },
+	{ _MMIO(0x9888), 0x02520033 },
+	{ _MMIO(0x9888), 0x0e534000 },
+	{ _MMIO(0x9888), 0x00534000 },
+	{ _MMIO(0x9888), 0x02531000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900800 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900062 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900c00 },
+	{ _MMIO(0x9888), 0x43900003 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+};
+
+static int
+get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_1;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_tdl_2[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0x00800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x00800000 },
+};
+
+static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00010003 },
+	{ _MMIO(0xe658), 0x00012011 },
+	{ _MMIO(0xe758), 0x00015014 },
+	{ _MMIO(0xe45c), 0x00051050 },
+	{ _MMIO(0xe55c), 0x00053052 },
+	{ _MMIO(0xe65c), 0x00055054 },
+};
+
+static const struct i915_oa_reg mux_config_tdl_2[] = {
+	{ _MMIO(0x9888), 0x12124d60 },
+	{ _MMIO(0x9888), 0x12322e60 },
+	{ _MMIO(0x9888), 0x12524d60 },
+	{ _MMIO(0x9888), 0x022f3000 },
+	{ _MMIO(0x9888), 0x0a4c0014 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x0c0fe000 },
+	{ _MMIO(0x9888), 0x0e0f0097 },
+	{ _MMIO(0x9888), 0x082c8000 },
+	{ _MMIO(0x9888), 0x0a2c8000 },
+	{ _MMIO(0x9888), 0x002d8000 },
+	{ _MMIO(0x9888), 0x062d4000 },
+	{ _MMIO(0x9888), 0x0410c000 },
+	{ _MMIO(0x9888), 0x0411c000 },
+	{ _MMIO(0x9888), 0x04121fb7 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x04135000 },
+	{ _MMIO(0x9888), 0x00308000 },
+	{ _MMIO(0x9888), 0x06304000 },
+	{ _MMIO(0x9888), 0x00318000 },
+	{ _MMIO(0x9888), 0x06314000 },
+	{ _MMIO(0x9888), 0x00321b80 },
+	{ _MMIO(0x9888), 0x0632003f },
+	{ _MMIO(0x9888), 0x00334000 },
+	{ _MMIO(0x9888), 0x06331000 },
+	{ _MMIO(0x9888), 0x0250c000 },
+	{ _MMIO(0x9888), 0x0251c000 },
+	{ _MMIO(0x9888), 0x02521fb7 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x02535000 },
+	{ _MMIO(0x9888), 0x1190fc00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x51900000 },
+	{ _MMIO(0x9888), 0x41900800 },
+	{ _MMIO(0x9888), 0x43900063 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900040 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
+		     const struct i915_oa_reg **regs,
+		     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_tdl_2;
+	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_compute_extra[] = {
+};
+
+static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
+};
+
+static const struct i915_oa_reg mux_config_compute_extra[] = {
+	{ _MMIO(0x9888), 0x121203e0 },
+	{ _MMIO(0x9888), 0x123203e0 },
+	{ _MMIO(0x9888), 0x125203e0 },
+	{ _MMIO(0x9888), 0x129203e0 },
+	{ _MMIO(0x9888), 0x12b203e0 },
+	{ _MMIO(0x9888), 0x12d203e0 },
+	{ _MMIO(0x9888), 0x131203e0 },
+	{ _MMIO(0x9888), 0x133203e0 },
+	{ _MMIO(0x9888), 0x135203e0 },
+	{ _MMIO(0x9888), 0x1a4ef000 },
+	{ _MMIO(0x9888), 0x1c4e0003 },
+	{ _MMIO(0x9888), 0x024ec000 },
+	{ _MMIO(0x9888), 0x044ec000 },
+	{ _MMIO(0x9888), 0x064ec000 },
+	{ _MMIO(0x9888), 0x022f4000 },
+	{ _MMIO(0x9888), 0x0c4c02a0 },
+	{ _MMIO(0x9888), 0x084ca000 },
+	{ _MMIO(0x9888), 0x0a4c0042 },
+	{ _MMIO(0x9888), 0x0c0d8000 },
+	{ _MMIO(0x9888), 0x0e0da000 },
+	{ _MMIO(0x9888), 0x000d8000 },
+	{ _MMIO(0x9888), 0x020da000 },
+	{ _MMIO(0x9888), 0x040da000 },
+	{ _MMIO(0x9888), 0x060d2000 },
+	{ _MMIO(0x9888), 0x100f0150 },
+	{ _MMIO(0x9888), 0x0c0f5000 },
+	{ _MMIO(0x9888), 0x0e0f006d },
+	{ _MMIO(0x9888), 0x182c00a8 },
+	{ _MMIO(0x9888), 0x022c8000 },
+	{ _MMIO(0x9888), 0x042c8000 },
+	{ _MMIO(0x9888), 0x062c8000 },
+	{ _MMIO(0x9888), 0x0c2c8000 },
+	{ _MMIO(0x9888), 0x042d8000 },
+	{ _MMIO(0x9888), 0x06104000 },
+	{ _MMIO(0x9888), 0x06114000 },
+	{ _MMIO(0x9888), 0x06120033 },
+	{ _MMIO(0x9888), 0x00120000 },
+	{ _MMIO(0x9888), 0x06131000 },
+	{ _MMIO(0x9888), 0x04308000 },
+	{ _MMIO(0x9888), 0x04318000 },
+	{ _MMIO(0x9888), 0x04321980 },
+	{ _MMIO(0x9888), 0x00320000 },
+	{ _MMIO(0x9888), 0x04334000 },
+	{ _MMIO(0x9888), 0x04504000 },
+	{ _MMIO(0x9888), 0x04514000 },
+	{ _MMIO(0x9888), 0x04520033 },
+	{ _MMIO(0x9888), 0x00520000 },
+	{ _MMIO(0x9888), 0x04531000 },
+	{ _MMIO(0x9888), 0x1acef000 },
+	{ _MMIO(0x9888), 0x1cce0003 },
+	{ _MMIO(0x9888), 0x00af8000 },
+	{ _MMIO(0x9888), 0x0ccc02a0 },
+	{ _MMIO(0x9888), 0x0acc0001 },
+	{ _MMIO(0x9888), 0x0c8d8000 },
+	{ _MMIO(0x9888), 0x0e8da000 },
+	{ _MMIO(0x9888), 0x008d8000 },
+	{ _MMIO(0x9888), 0x028da000 },
+	{ _MMIO(0x9888), 0x108f0150 },
+	{ _MMIO(0x9888), 0x0c8fb000 },
+	{ _MMIO(0x9888), 0x0e8f0001 },
+	{ _MMIO(0x9888), 0x18ac00a8 },
+	{ _MMIO(0x9888), 0x06ac8000 },
+	{ _MMIO(0x9888), 0x02ad4000 },
+	{ _MMIO(0x9888), 0x02908000 },
+	{ _MMIO(0x9888), 0x02918000 },
+	{ _MMIO(0x9888), 0x02921980 },
+	{ _MMIO(0x9888), 0x00920000 },
+	{ _MMIO(0x9888), 0x02934000 },
+	{ _MMIO(0x9888), 0x02b04000 },
+	{ _MMIO(0x9888), 0x02b14000 },
+	{ _MMIO(0x9888), 0x02b20033 },
+	{ _MMIO(0x9888), 0x00b20000 },
+	{ _MMIO(0x9888), 0x02b31000 },
+	{ _MMIO(0x9888), 0x00d08000 },
+	{ _MMIO(0x9888), 0x00d18000 },
+	{ _MMIO(0x9888), 0x00d21980 },
+	{ _MMIO(0x9888), 0x00d34000 },
+	{ _MMIO(0x9888), 0x072f8000 },
+	{ _MMIO(0x9888), 0x0d4c0100 },
+	{ _MMIO(0x9888), 0x0d0d8000 },
+	{ _MMIO(0x9888), 0x0f0da000 },
+	{ _MMIO(0x9888), 0x110f01b0 },
+	{ _MMIO(0x9888), 0x192c0080 },
+	{ _MMIO(0x9888), 0x0f2d4000 },
+	{ _MMIO(0x9888), 0x0f108000 },
+	{ _MMIO(0x9888), 0x0f118000 },
+	{ _MMIO(0x9888), 0x0f121980 },
+	{ _MMIO(0x9888), 0x01120000 },
+	{ _MMIO(0x9888), 0x0f134000 },
+	{ _MMIO(0x9888), 0x0f304000 },
+	{ _MMIO(0x9888), 0x0f314000 },
+	{ _MMIO(0x9888), 0x0f320033 },
+	{ _MMIO(0x9888), 0x01320000 },
+	{ _MMIO(0x9888), 0x0f331000 },
+	{ _MMIO(0x9888), 0x0d508000 },
+	{ _MMIO(0x9888), 0x0d518000 },
+	{ _MMIO(0x9888), 0x0d521980 },
+	{ _MMIO(0x9888), 0x01520000 },
+	{ _MMIO(0x9888), 0x0d534000 },
+	{ _MMIO(0x9888), 0x1190ff80 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900c00 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+	{ _MMIO(0x9888), 0x4b900002 },
+	{ _MMIO(0x9888), 0x59900000 },
+	{ _MMIO(0x9888), 0x51901100 },
+	{ _MMIO(0x9888), 0x41901000 },
+	{ _MMIO(0x9888), 0x43901423 },
+	{ _MMIO(0x9888), 0x53903331 },
+	{ _MMIO(0x9888), 0x45900044 },
+};
+
+static int
+get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
+			     const struct i915_oa_reg **regs,
+			     int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_compute_extra;
+	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0x30800000 },
+	{ _MMIO(0x2770), 0x00100030 },
+	{ _MMIO(0x2774), 0x0000fff9 },
+	{ _MMIO(0x2778), 0x00000002 },
+	{ _MMIO(0x277c), 0x0000fffc },
+	{ _MMIO(0x2780), 0x00000002 },
+	{ _MMIO(0x2784), 0x0000fff3 },
+	{ _MMIO(0x2788), 0x00100180 },
+	{ _MMIO(0x278c), 0x0000ffcf },
+	{ _MMIO(0x2790), 0x00000002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00000002 },
+	{ _MMIO(0x279c), 0x0000ff3f },
+};
+
+static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
+	{ _MMIO(0xe458), 0x00005004 },
+	{ _MMIO(0xe558), 0x00008003 },
+};
+
+static const struct i915_oa_reg mux_config_vme_pipe[] = {
+	{ _MMIO(0x9888), 0x141a5800 },
+	{ _MMIO(0x9888), 0x161a00c0 },
+	{ _MMIO(0x9888), 0x12180240 },
+	{ _MMIO(0x9888), 0x14180002 },
+	{ _MMIO(0x9888), 0x149a5800 },
+	{ _MMIO(0x9888), 0x169a00c0 },
+	{ _MMIO(0x9888), 0x12980240 },
+	{ _MMIO(0x9888), 0x14980002 },
+	{ _MMIO(0x9888), 0x1a4e3fc0 },
+	{ _MMIO(0x9888), 0x002f1000 },
+	{ _MMIO(0x9888), 0x022f8000 },
+	{ _MMIO(0x9888), 0x042f3000 },
+	{ _MMIO(0x9888), 0x004c4000 },
+	{ _MMIO(0x9888), 0x0a4c9500 },
+	{ _MMIO(0x9888), 0x0c4c002a },
+	{ _MMIO(0x9888), 0x000d2000 },
+	{ _MMIO(0x9888), 0x060d8000 },
+	{ _MMIO(0x9888), 0x080da000 },
+	{ _MMIO(0x9888), 0x0a0da000 },
+	{ _MMIO(0x9888), 0x0c0da000 },
+	{ _MMIO(0x9888), 0x0c0f0400 },
+	{ _MMIO(0x9888), 0x0e0f5500 },
+	{ _MMIO(0x9888), 0x100f0015 },
+	{ _MMIO(0x9888), 0x002c8000 },
+	{ _MMIO(0x9888), 0x0e2c8000 },
+	{ _MMIO(0x9888), 0x162caa00 },
+	{ _MMIO(0x9888), 0x182c000a },
+	{ _MMIO(0x9888), 0x04193000 },
+	{ _MMIO(0x9888), 0x081a28c1 },
+	{ _MMIO(0x9888), 0x001a0000 },
+	{ _MMIO(0x9888), 0x00133000 },
+	{ _MMIO(0x9888), 0x0613c000 },
+	{ _MMIO(0x9888), 0x0813f000 },
+	{ _MMIO(0x9888), 0x00172000 },
+	{ _MMIO(0x9888), 0x06178000 },
+	{ _MMIO(0x9888), 0x0817a000 },
+	{ _MMIO(0x9888), 0x00180037 },
+	{ _MMIO(0x9888), 0x06180940 },
+	{ _MMIO(0x9888), 0x08180000 },
+	{ _MMIO(0x9888), 0x02180000 },
+	{ _MMIO(0x9888), 0x04183000 },
+	{ _MMIO(0x9888), 0x04afc000 },
+	{ _MMIO(0x9888), 0x06af3000 },
+	{ _MMIO(0x9888), 0x0acc4000 },
+	{ _MMIO(0x9888), 0x0ccc0015 },
+	{ _MMIO(0x9888), 0x0a8da000 },
+	{ _MMIO(0x9888), 0x0c8da000 },
+	{ _MMIO(0x9888), 0x0e8f4000 },
+	{ _MMIO(0x9888), 0x108f0015 },
+	{ _MMIO(0x9888), 0x16aca000 },
+	{ _MMIO(0x9888), 0x18ac000a },
+	{ _MMIO(0x9888), 0x06993000 },
+	{ _MMIO(0x9888), 0x0c9a28c1 },
+	{ _MMIO(0x9888), 0x009a0000 },
+	{ _MMIO(0x9888), 0x0a93f000 },
+	{ _MMIO(0x9888), 0x0c93f000 },
+	{ _MMIO(0x9888), 0x0a97a000 },
+	{ _MMIO(0x9888), 0x0c97a000 },
+	{ _MMIO(0x9888), 0x0a980977 },
+	{ _MMIO(0x9888), 0x08980000 },
+	{ _MMIO(0x9888), 0x04980000 },
+	{ _MMIO(0x9888), 0x06983000 },
+	{ _MMIO(0x9888), 0x119000ff },
+	{ _MMIO(0x9888), 0x51900010 },
+	{ _MMIO(0x9888), 0x41900060 },
+	{ _MMIO(0x9888), 0x55900111 },
+	{ _MMIO(0x9888), 0x45900c00 },
+	{ _MMIO(0x9888), 0x47900821 },
+	{ _MMIO(0x9888), 0x57900000 },
+	{ _MMIO(0x9888), 0x49900002 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
+			const struct i915_oa_reg **regs,
+			int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_vme_pipe;
+	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
+	n++;
+
+	return n;
+}
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9888), 0x11810000 },
+	{ _MMIO(0x9888), 0x07810013 },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930040 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static int
+get_test_oa_mux_config(struct drm_i915_private *dev_priv,
+		       const struct i915_oa_reg **regs,
+		       int *lens)
+{
+	int n = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
+	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
+
+	regs[n] = mux_config_test_oa;
+	lens[n] = ARRAY_SIZE(mux_config_test_oa);
+	n++;
+
+	return n;
+}
+
+int i915_oa_select_metric_set_sklgt4(struct drm_i915_private *dev_priv)
+{
+	dev_priv->perf.oa.n_mux_configs = 0;
+	dev_priv->perf.oa.b_counter_regs = NULL;
+	dev_priv->perf.oa.b_counter_regs_len = 0;
+	dev_priv->perf.oa.flex_regs = NULL;
+	dev_priv->perf.oa.flex_regs_len = 0;
+
+	switch (dev_priv->perf.oa.metrics_set) {
+	case METRIC_SET_ID_RENDER_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_basic_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_basic);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_BASIC:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_basic_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_basic;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_basic);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_basic;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_basic);
+
+		return 0;
+	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_render_pipe_profile_mux_config(dev_priv,
+							   dev_priv->perf.oa.mux_regs,
+							   dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_render_pipe_profile;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_render_pipe_profile);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_render_pipe_profile;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_READS:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_reads_mux_config(dev_priv,
+						    dev_priv->perf.oa.mux_regs,
+						    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_reads;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_reads);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_reads;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_reads);
+
+		return 0;
+	case METRIC_SET_ID_MEMORY_WRITES:
+		dev_priv->perf.oa.n_mux_configs =
+			get_memory_writes_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_memory_writes;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_memory_writes);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_memory_writes;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_memory_writes);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTENDED:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extended_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extended;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extended);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extended;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extended);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_L3_CACHE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_l3_cache_mux_config(dev_priv,
+							dev_priv->perf.oa.mux_regs,
+							dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_l3_cache;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_l3_cache);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_l3_cache;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
+
+		return 0;
+	case METRIC_SET_ID_HDC_AND_SF:
+		dev_priv->perf.oa.n_mux_configs =
+			get_hdc_and_sf_mux_config(dev_priv,
+						  dev_priv->perf.oa.mux_regs,
+						  dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_hdc_and_sf;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_hdc_and_sf);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_hdc_and_sf;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
+
+		return 0;
+	case METRIC_SET_ID_L3_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_1_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_1);
+
+		return 0;
+	case METRIC_SET_ID_L3_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_2_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_2);
+
+		return 0;
+	case METRIC_SET_ID_L3_3:
+		dev_priv->perf.oa.n_mux_configs =
+			get_l3_3_mux_config(dev_priv,
+					    dev_priv->perf.oa.mux_regs,
+					    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_l3_3;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_l3_3);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_l3_3;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_l3_3);
+
+		return 0;
+	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
+		dev_priv->perf.oa.n_mux_configs =
+			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
+								    dev_priv->perf.oa.mux_regs,
+								    dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_rasterizer_and_pixel_backend;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
+
+		return 0;
+	case METRIC_SET_ID_SAMPLER:
+		dev_priv->perf.oa.n_mux_configs =
+			get_sampler_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_sampler;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_sampler);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_sampler;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_sampler);
+
+		return 0;
+	case METRIC_SET_ID_TDL_1:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_1_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_1;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_1);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_1;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_1);
+
+		return 0;
+	case METRIC_SET_ID_TDL_2:
+		dev_priv->perf.oa.n_mux_configs =
+			get_tdl_2_mux_config(dev_priv,
+					     dev_priv->perf.oa.mux_regs,
+					     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_tdl_2;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_tdl_2);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_tdl_2;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_tdl_2);
+
+		return 0;
+	case METRIC_SET_ID_COMPUTE_EXTRA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_compute_extra_mux_config(dev_priv,
+						     dev_priv->perf.oa.mux_regs,
+						     dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_compute_extra;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_compute_extra);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_compute_extra;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_compute_extra);
+
+		return 0;
+	case METRIC_SET_ID_VME_PIPE:
+		dev_priv->perf.oa.n_mux_configs =
+			get_vme_pipe_mux_config(dev_priv,
+						dev_priv->perf.oa.mux_regs,
+						dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_vme_pipe;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_vme_pipe);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_vme_pipe;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_vme_pipe);
+
+		return 0;
+	case METRIC_SET_ID_TEST_OA:
+		dev_priv->perf.oa.n_mux_configs =
+			get_test_oa_mux_config(dev_priv,
+					       dev_priv->perf.oa.mux_regs,
+					       dev_priv->perf.oa.mux_regs_lens);
+		if (dev_priv->perf.oa.n_mux_configs == 0) {
+			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
+
+			/* EINVAL because *_register_sysfs already checked this
+			 * and so it wouldn't have been advertised to userspace and
+			 * so shouldn't have been requested
+			 */
+			return -EINVAL;
+		}
+
+		dev_priv->perf.oa.b_counter_regs =
+			b_counter_config_test_oa;
+		dev_priv->perf.oa.b_counter_regs_len =
+			ARRAY_SIZE(b_counter_config_test_oa);
+
+		dev_priv->perf.oa.flex_regs =
+			flex_eu_config_test_oa;
+		dev_priv->perf.oa.flex_regs_len =
+			ARRAY_SIZE(flex_eu_config_test_oa);
+
+		return 0;
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t
+show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
+}
+
+static struct device_attribute dev_attr_render_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_basic[] = {
+	&dev_attr_render_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_basic = {
+	.name = "bad77c24-cc64-480d-99bf-e7b740713800",
+	.attrs =  attrs_render_basic,
+};
+
+static ssize_t
+show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
+}
+
+static struct device_attribute dev_attr_compute_basic_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_basic_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_basic[] = {
+	&dev_attr_compute_basic_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_basic = {
+	.name = "7277228f-e7f3-4743-945a-6a2049d11377",
+	.attrs =  attrs_compute_basic,
+};
+
+static ssize_t
+show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
+}
+
+static struct device_attribute dev_attr_render_pipe_profile_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_render_pipe_profile_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_render_pipe_profile[] = {
+	&dev_attr_render_pipe_profile_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_render_pipe_profile = {
+	.name = "463c668c-3f60-49b6-8f85-d995b635b3b2",
+	.attrs =  attrs_render_pipe_profile,
+};
+
+static ssize_t
+show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
+}
+
+static struct device_attribute dev_attr_memory_reads_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_reads_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_reads[] = {
+	&dev_attr_memory_reads_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_reads = {
+	.name = "3ae6e74c-72c3-4040-9bd0-7961430b8cc8",
+	.attrs =  attrs_memory_reads,
+};
+
+static ssize_t
+show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
+}
+
+static struct device_attribute dev_attr_memory_writes_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_memory_writes_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_memory_writes[] = {
+	&dev_attr_memory_writes_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_memory_writes = {
+	.name = "055f256d-4052-467c-8dec-6064a4806433",
+	.attrs =  attrs_memory_writes,
+};
+
+static ssize_t
+show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+}
+
+static struct device_attribute dev_attr_compute_extended_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extended_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extended[] = {
+	&dev_attr_compute_extended_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extended = {
+	.name = "753972d4-87cd-4460-824d-754463ac5054",
+	.attrs =  attrs_compute_extended,
+};
+
+static ssize_t
+show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
+}
+
+static struct device_attribute dev_attr_compute_l3_cache_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_l3_cache_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_l3_cache[] = {
+	&dev_attr_compute_l3_cache_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_l3_cache = {
+	.name = "4e4392e9-8f73-457b-ab44-b49f7a0c733b",
+	.attrs =  attrs_compute_l3_cache,
+};
+
+static ssize_t
+show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
+}
+
+static struct device_attribute dev_attr_hdc_and_sf_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_hdc_and_sf_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_hdc_and_sf[] = {
+	&dev_attr_hdc_and_sf_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_hdc_and_sf = {
+	.name = "730d95dd-7da8-4e1c-ab8d-c0eb1e4c1805",
+	.attrs =  attrs_hdc_and_sf,
+};
+
+static ssize_t
+show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
+}
+
+static struct device_attribute dev_attr_l3_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_1[] = {
+	&dev_attr_l3_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_1 = {
+	.name = "d9e86d70-462b-462a-851e-fd63e8c13d63",
+	.attrs =  attrs_l3_1,
+};
+
+static ssize_t
+show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
+}
+
+static struct device_attribute dev_attr_l3_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_2[] = {
+	&dev_attr_l3_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_2 = {
+	.name = "52200424-6ee9-48b3-b7fa-0afcf1975e4d",
+	.attrs =  attrs_l3_2,
+};
+
+static ssize_t
+show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
+}
+
+static struct device_attribute dev_attr_l3_3_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_l3_3_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_l3_3[] = {
+	&dev_attr_l3_3_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_l3_3 = {
+	.name = "1988315f-0a26-44df-acb0-df7ec86b1456",
+	.attrs =  attrs_l3_3,
+};
+
+static ssize_t
+show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
+}
+
+static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_rasterizer_and_pixel_backend_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
+	&dev_attr_rasterizer_and_pixel_backend_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_rasterizer_and_pixel_backend = {
+	.name = "f1f17ca7-286e-4ae5-9d15-9fccad6c665d",
+	.attrs =  attrs_rasterizer_and_pixel_backend,
+};
+
+static ssize_t
+show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
+}
+
+static struct device_attribute dev_attr_sampler_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_sampler_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_sampler[] = {
+	&dev_attr_sampler_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_sampler = {
+	.name = "00a9e0fb-3d2e-4405-852c-dce6334ffb3b",
+	.attrs =  attrs_sampler,
+};
+
+static ssize_t
+show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
+}
+
+static struct device_attribute dev_attr_tdl_1_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_1_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_1[] = {
+	&dev_attr_tdl_1_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_1 = {
+	.name = "13dcc50a-7ec0-409b-99d6-a3f932cedcb3",
+	.attrs =  attrs_tdl_1,
+};
+
+static ssize_t
+show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
+}
+
+static struct device_attribute dev_attr_tdl_2_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_tdl_2_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_tdl_2[] = {
+	&dev_attr_tdl_2_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_tdl_2 = {
+	.name = "97875e21-6624-4aee-9191-682feb3eae21",
+	.attrs =  attrs_tdl_2,
+};
+
+static ssize_t
+show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
+}
+
+static struct device_attribute dev_attr_compute_extra_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_compute_extra_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_compute_extra[] = {
+	&dev_attr_compute_extra_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_compute_extra = {
+	.name = "a5aa857d-e8f0-4dfa-8981-ce340fa748fd",
+	.attrs =  attrs_compute_extra,
+};
+
+static ssize_t
+show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
+}
+
+static struct device_attribute dev_attr_vme_pipe_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_vme_pipe_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_vme_pipe[] = {
+	&dev_attr_vme_pipe_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_vme_pipe = {
+	.name = "0e8d8b86-4ee7-4cdd-aaaa-58adc92cb29e",
+	.attrs =  attrs_vme_pipe,
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+}
+
+static struct device_attribute dev_attr_test_oa_id = {
+	.attr = { .name = "id", .mode = 0444 },
+	.show = show_test_oa_id,
+	.store = NULL,
+};
+
+static struct attribute *attrs_test_oa[] = {
+	&dev_attr_test_oa_id.attr,
+	NULL,
+};
+
+static struct attribute_group group_test_oa = {
+	.name = "882fa433-1f4a-4a67-a962-c741888fe5f5",
+	.attrs =  attrs_test_oa,
+};
+
+int
+i915_perf_register_sysfs_sklgt4(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	int ret = 0;
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+		if (ret)
+			goto error_render_basic;
+	}
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+		if (ret)
+			goto error_compute_basic;
+	}
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+		if (ret)
+			goto error_render_pipe_profile;
+	}
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+		if (ret)
+			goto error_memory_reads;
+	}
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+		if (ret)
+			goto error_memory_writes;
+	}
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+		if (ret)
+			goto error_compute_extended;
+	}
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+		if (ret)
+			goto error_compute_l3_cache;
+	}
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+		if (ret)
+			goto error_hdc_and_sf;
+	}
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+		if (ret)
+			goto error_l3_1;
+	}
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+		if (ret)
+			goto error_l3_2;
+	}
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+		if (ret)
+			goto error_l3_3;
+	}
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+		if (ret)
+			goto error_rasterizer_and_pixel_backend;
+	}
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
+		if (ret)
+			goto error_sampler;
+	}
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+		if (ret)
+			goto error_tdl_1;
+	}
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+		if (ret)
+			goto error_tdl_2;
+	}
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+		if (ret)
+			goto error_compute_extra;
+	}
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+		if (ret)
+			goto error_vme_pipe;
+	}
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
+		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+		if (ret)
+			goto error_test_oa;
+	}
+
+	return 0;
+
+error_test_oa:
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+error_vme_pipe:
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+error_compute_extra:
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+error_tdl_2:
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+error_tdl_1:
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+error_sampler:
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+error_rasterizer_and_pixel_backend:
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+error_l3_3:
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+error_l3_2:
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+error_l3_1:
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+error_hdc_and_sf:
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+error_compute_l3_cache:
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+error_compute_extended:
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+error_memory_writes:
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+error_memory_reads:
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+error_render_pipe_profile:
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+error_compute_basic:
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+error_render_basic:
+	return ret;
+}
+
+void
+i915_perf_unregister_sysfs_sklgt4(struct drm_i915_private *dev_priv)
+{
+	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
+	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+
+	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
+	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
+	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
+	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
+	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
+	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
+	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
+	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
+	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
+	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
+	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
+	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
+	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
+	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
+	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
+	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
+	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
+	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
+		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.h b/drivers/gpu/drm/i915/i915_oa_sklgt4.h
new file mode 100644
index 000000000000..1b718f15f62e
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.h
@@ -0,0 +1,40 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_SKLGT4_H__
+#define __I915_OA_SKLGT4_H__
+
+extern int i915_oa_n_builtin_metric_sets_sklgt4;
+
+extern int i915_oa_select_metric_set_sklgt4(struct drm_i915_private *dev_priv);
+
+extern int i915_perf_register_sysfs_sklgt4(struct drm_i915_private *dev_priv);
+
+extern void i915_perf_unregister_sysfs_sklgt4(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 1a78363c7f4a..506ec32b9e53 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -220,7 +220,6 @@ static const struct intel_device_info intel_ironlake_m_info = {
 	.has_rc6 = 1, \
 	.has_rc6p = 1, \
 	.has_gmbus_irq = 1, \
-	.has_hw_contexts = 1, \
 	.has_aliasing_ppgtt = 1, \
 	GEN_DEFAULT_PIPEOFFSETS, \
 	CURSOR_OFFSETS
@@ -245,7 +244,6 @@ static const struct intel_device_info intel_sandybridge_m_info = {
 	.has_rc6 = 1, \
 	.has_rc6p = 1, \
 	.has_gmbus_irq = 1, \
-	.has_hw_contexts = 1, \
 	.has_aliasing_ppgtt = 1, \
 	.has_full_ppgtt = 1, \
 	GEN_DEFAULT_PIPEOFFSETS, \
@@ -280,7 +278,6 @@ static const struct intel_device_info intel_valleyview_info = {
 	.has_runtime_pm = 1,
 	.has_rc6 = 1,
 	.has_gmbus_irq = 1,
-	.has_hw_contexts = 1,
 	.has_gmch_display = 1,
 	.has_hotplug = 1,
 	.has_aliasing_ppgtt = 1,
@@ -315,16 +312,17 @@ static const struct intel_device_info intel_haswell_info = {
 	.has_full_48bit_ppgtt = 1, \
 	.has_64bit_reloc = 1
 
+#define BDW_PLATFORM \
+	BDW_FEATURES, \
+	.gen = 8, \
+	.platform = INTEL_BROADWELL
+
 static const struct intel_device_info intel_broadwell_info = {
-	BDW_FEATURES,
-	.gen = 8,
-	.platform = INTEL_BROADWELL,
+	BDW_PLATFORM,
 };
 
 static const struct intel_device_info intel_broadwell_gt3_info = {
-	BDW_FEATURES,
-	.gen = 8,
-	.platform = INTEL_BROADWELL,
+	BDW_PLATFORM,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
@@ -340,7 +338,6 @@ static const struct intel_device_info intel_cherryview_info = {
 	.has_resource_streamer = 1,
 	.has_rc6 = 1,
 	.has_gmbus_irq = 1,
-	.has_hw_contexts = 1,
 	.has_logical_ring_contexts = 1,
 	.has_gmch_display = 1,
 	.has_aliasing_ppgtt = 1,
@@ -351,22 +348,20 @@ static const struct intel_device_info intel_cherryview_info = {
 	CHV_COLORS,
 };
 
+#define SKL_PLATFORM \
+	BDW_FEATURES, \
+	.gen = 9, \
+	.platform = INTEL_SKYLAKE, \
+	.has_csr = 1, \
+	.has_guc = 1, \
+	.ddb_size = 896
+
 static const struct intel_device_info intel_skylake_info = {
-	BDW_FEATURES,
-	.platform = INTEL_SKYLAKE,
-	.gen = 9,
-	.has_csr = 1,
-	.has_guc = 1,
-	.ddb_size = 896,
+	SKL_PLATFORM,
 };
 
 static const struct intel_device_info intel_skylake_gt3_info = {
-	BDW_FEATURES,
-	.platform = INTEL_SKYLAKE,
-	.gen = 9,
-	.has_csr = 1,
-	.has_guc = 1,
-	.ddb_size = 896,
+	SKL_PLATFORM,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
@@ -387,7 +382,6 @@ static const struct intel_device_info intel_skylake_gt3_info = {
 	.has_rc6 = 1, \
 	.has_dp_mst = 1, \
 	.has_gmbus_irq = 1, \
-	.has_hw_contexts = 1, \
 	.has_logical_ring_contexts = 1, \
 	.has_guc = 1, \
 	.has_aliasing_ppgtt = 1, \
@@ -406,28 +400,52 @@ static const struct intel_device_info intel_broxton_info = {
 static const struct intel_device_info intel_geminilake_info = {
 	GEN9_LP_FEATURES,
 	.platform = INTEL_GEMINILAKE,
-	.is_alpha_support = 1,
 	.ddb_size = 1024,
 	.color = { .degamma_lut_size = 0, .gamma_lut_size = 1024 }
 };
 
+#define KBL_PLATFORM \
+	BDW_FEATURES, \
+	.gen = 9, \
+	.platform = INTEL_KABYLAKE, \
+	.has_csr = 1, \
+	.has_guc = 1, \
+	.ddb_size = 896
+
 static const struct intel_device_info intel_kabylake_info = {
-	BDW_FEATURES,
-	.platform = INTEL_KABYLAKE,
-	.gen = 9,
-	.has_csr = 1,
-	.has_guc = 1,
-	.ddb_size = 896,
+	KBL_PLATFORM,
 };
 
 static const struct intel_device_info intel_kabylake_gt3_info = {
+	KBL_PLATFORM,
+	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
+#define CFL_PLATFORM \
+	.is_alpha_support = 1, \
+	BDW_FEATURES, \
+	.gen = 9, \
+	.platform = INTEL_COFFEELAKE, \
+	.has_csr = 1, \
+	.has_guc = 1, \
+	.ddb_size = 896
+
+static const struct intel_device_info intel_coffeelake_info = {
+	CFL_PLATFORM,
+};
+
+static const struct intel_device_info intel_coffeelake_gt3_info = {
+	CFL_PLATFORM,
+	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
+static const struct intel_device_info intel_cannonlake_info = {
 	BDW_FEATURES,
-	.platform = INTEL_KABYLAKE,
-	.gen = 9,
+	.is_alpha_support = 1,
+	.platform = INTEL_CANNONLAKE,
+	.gen = 10,
+	.ddb_size = 1024,
 	.has_csr = 1,
-	.has_guc = 1,
-	.ddb_size = 896,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
 /*
@@ -474,6 +492,10 @@ static const struct pci_device_id pciidlist[] = {
 	INTEL_KBL_GT2_IDS(&intel_kabylake_info),
 	INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info),
 	INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info),
+	INTEL_CFL_S_IDS(&intel_coffeelake_info),
+	INTEL_CFL_H_IDS(&intel_coffeelake_info),
+	INTEL_CFL_U_IDS(&intel_coffeelake_gt3_info),
+	INTEL_CNL_IDS(&intel_cannonlake_info),
 	{0, 0, 0}
 };
 MODULE_DEVICE_TABLE(pci, pciidlist);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 060b171480d5..38c44407bafc 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -196,6 +196,15 @@
 
 #include "i915_drv.h"
 #include "i915_oa_hsw.h"
+#include "i915_oa_bdw.h"
+#include "i915_oa_chv.h"
+#include "i915_oa_sklgt2.h"
+#include "i915_oa_sklgt3.h"
+#include "i915_oa_sklgt4.h"
+#include "i915_oa_bxt.h"
+#include "i915_oa_kblgt2.h"
+#include "i915_oa_kblgt3.h"
+#include "i915_oa_glk.h"
 
 /* HW requires this to be a power of two, between 128k and 16M, though driver
  * is currently generally designed assuming the largest 16M size is used such
@@ -205,25 +214,49 @@
 
 #define OA_TAKEN(tail, head)	((tail - head) & (OA_BUFFER_SIZE - 1))
 
-/* There's a HW race condition between OA unit tail pointer register updates and
+/**
+ * DOC: OA Tail Pointer Race
+ *
+ * There's a HW race condition between OA unit tail pointer register updates and
  * writes to memory whereby the tail pointer can sometimes get ahead of what's
- * been written out to the OA buffer so far.
+ * been written out to the OA buffer so far (in terms of what's visible to the
+ * CPU).
+ *
+ * Although this can be observed explicitly while copying reports to userspace
+ * by checking for a zeroed report-id field in tail reports, we want to account
+ * for this earlier, as part of the oa_buffer_check to avoid lots of redundant
+ * read() attempts.
+ *
+ * In effect we define a tail pointer for reading that lags the real tail
+ * pointer by at least %OA_TAIL_MARGIN_NSEC nanoseconds, which gives enough
+ * time for the corresponding reports to become visible to the CPU.
  *
- * Although this can be observed explicitly by checking for a zeroed report-id
- * field in tail reports, it seems preferable to account for this earlier e.g.
- * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles
- * in this situation.
+ * To manage this we actually track two tail pointers:
+ *  1) An 'aging' tail with an associated timestamp that is tracked until we
+ *     can trust the corresponding data is visible to the CPU; at which point
+ *     it is considered 'aged'.
+ *  2) An 'aged' tail that can be used for read()ing.
  *
- * To give time for the most recent reports to land before they may be copied to
- * userspace, the driver operates as if the tail pointer effectively lags behind
- * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated
- * based on this constant in nanoseconds, the current OA sampling exponent
- * and current report size.
+ * The two separate pointers let us decouple read()s from tail pointer aging.
  *
- * There is also a fallback check while reading to simply skip over reports with
- * a zeroed report-id.
+ * The tail pointers are checked and updated at a limited rate within a hrtimer
+ * callback (the same callback that is used for delivering POLLIN events)
+ *
+ * Initially the tails are marked invalid with %INVALID_TAIL_PTR which
+ * indicates that an updated tail pointer is needed.
+ *
+ * Most of the implementation details for this workaround are in
+ * oa_buffer_check_unlocked() and _append_oa_reports()
+ *
+ * Note for posterity: previously the driver used to define an effective tail
+ * pointer that lagged the real pointer by a 'tail margin' measured in bytes
+ * derived from %OA_TAIL_MARGIN_NSEC and the configured sampling frequency.
+ * This was flawed considering that the OA unit may also automatically generate
+ * non-periodic reports (such as on context switch) or the OA unit may be
+ * enabled without any periodic sampling.
  */
 #define OA_TAIL_MARGIN_NSEC	100000ULL
+#define INVALID_TAIL_PTR	0xffffffff
 
 /* frequency for checking whether the OA unit has written new reports to the
  * circular OA buffer...
@@ -248,13 +281,22 @@ static u32 i915_perf_stream_paranoid = true;
 
 #define INVALID_CTX_ID 0xffffffff
 
+/* On Gen8+ automatically triggered OA reports include a 'reason' field... */
+#define OAREPORT_REASON_MASK           0x3f
+#define OAREPORT_REASON_SHIFT          19
+#define OAREPORT_REASON_TIMER          (1<<0)
+#define OAREPORT_REASON_CTX_SWITCH     (1<<3)
+#define OAREPORT_REASON_CLK_RATIO      (1<<5)
+
 
 /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate
  *
- * 160ns is the smallest sampling period we can theoretically program the OA
- * unit with on Haswell, corresponding to 6.25MHz.
+ * The highest sampling frequency we can theoretically program the OA unit
+ * with is always half the timestamp frequency: E.g. 6.25Mhz for Haswell.
+ *
+ * Initialized just before we register the sysctl parameter.
  */
-static int oa_sample_rate_hard_limit = 6250000;
+static int oa_sample_rate_hard_limit;
 
 /* Theoretically we can program the OA unit to sample every 160ns but don't
  * allow that by default unless root...
@@ -279,6 +321,13 @@ static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = {
 	[I915_OA_FORMAT_C4_B8]	    = { 7, 64 },
 };
 
+static struct i915_oa_format gen8_plus_oa_formats[I915_OA_FORMAT_MAX] = {
+	[I915_OA_FORMAT_A12]		    = { 0, 64 },
+	[I915_OA_FORMAT_A12_B8_C8]	    = { 2, 128 },
+	[I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 },
+	[I915_OA_FORMAT_C4_B8]		    = { 7, 64 },
+};
+
 #define SAMPLE_OA_REPORT      (1<<0)
 
 /**
@@ -308,27 +357,132 @@ struct perf_open_properties {
 	int oa_period_exponent;
 };
 
-/* NB: This is either called via fops or the poll check hrtimer (atomic ctx)
+static u32 gen8_oa_hw_tail_read(struct drm_i915_private *dev_priv)
+{
+	return I915_READ(GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK;
+}
+
+static u32 gen7_oa_hw_tail_read(struct drm_i915_private *dev_priv)
+{
+	u32 oastatus1 = I915_READ(GEN7_OASTATUS1);
+
+	return oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
+}
+
+/**
+ * oa_buffer_check_unlocked - check for data and update tail ptr state
+ * @dev_priv: i915 device instance
+ *
+ * This is either called via fops (for blocking reads in user ctx) or the poll
+ * check hrtimer (atomic ctx) to check the OA buffer tail pointer and check
+ * if there is data available for userspace to read.
+ *
+ * This function is central to providing a workaround for the OA unit tail
+ * pointer having a race with respect to what data is visible to the CPU.
+ * It is responsible for reading tail pointers from the hardware and giving
+ * the pointers time to 'age' before they are made available for reading.
+ * (See description of OA_TAIL_MARGIN_NSEC above for further details.)
  *
- * It's safe to read OA config state here unlocked, assuming that this is only
- * called while the stream is enabled, while the global OA configuration can't
- * be modified.
+ * Besides returning true when there is data available to read() this function
+ * also has the side effect of updating the oa_buffer.tails[], .aging_timestamp
+ * and .aged_tail_idx state used for reading.
  *
- * Note: we don't lock around the head/tail reads even though there's the slim
- * possibility of read() fop errors forcing a re-init of the OA buffer
- * pointers.  A race here could result in a false positive !empty status which
- * is acceptable.
+ * Note: It's safe to read OA config state here unlocked, assuming that this is
+ * only called while the stream is enabled, while the global OA configuration
+ * can't be modified.
+ *
+ * Returns: %true if the OA buffer contains data, else %false
  */
-static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv)
+static bool oa_buffer_check_unlocked(struct drm_i915_private *dev_priv)
 {
 	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
-	u32 oastatus2 = I915_READ(GEN7_OASTATUS2);
-	u32 oastatus1 = I915_READ(GEN7_OASTATUS1);
-	u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK;
-	u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
+	unsigned long flags;
+	unsigned int aged_idx;
+	u32 head, hw_tail, aged_tail, aging_tail;
+	u64 now;
+
+	/* We have to consider the (unlikely) possibility that read() errors
+	 * could result in an OA buffer reset which might reset the head,
+	 * tails[] and aged_tail state.
+	 */
+	spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	/* NB: The head we observe here might effectively be a little out of
+	 * date (between head and tails[aged_idx].offset if there is currently
+	 * a read() in progress.
+	 */
+	head = dev_priv->perf.oa.oa_buffer.head;
 
-	return OA_TAKEN(tail, head) <
-		dev_priv->perf.oa.tail_margin + report_size;
+	aged_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx;
+	aged_tail = dev_priv->perf.oa.oa_buffer.tails[aged_idx].offset;
+	aging_tail = dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset;
+
+	hw_tail = dev_priv->perf.oa.ops.oa_hw_tail_read(dev_priv);
+
+	/* The tail pointer increases in 64 byte increments,
+	 * not in report_size steps...
+	 */
+	hw_tail &= ~(report_size - 1);
+
+	now = ktime_get_mono_fast_ns();
+
+	/* Update the aged tail
+	 *
+	 * Flip the tail pointer available for read()s once the aging tail is
+	 * old enough to trust that the corresponding data will be visible to
+	 * the CPU...
+	 *
+	 * Do this before updating the aging pointer in case we may be able to
+	 * immediately start aging a new pointer too (if new data has become
+	 * available) without needing to wait for a later hrtimer callback.
+	 */
+	if (aging_tail != INVALID_TAIL_PTR &&
+	    ((now - dev_priv->perf.oa.oa_buffer.aging_timestamp) >
+	     OA_TAIL_MARGIN_NSEC)) {
+
+		aged_idx ^= 1;
+		dev_priv->perf.oa.oa_buffer.aged_tail_idx = aged_idx;
+
+		aged_tail = aging_tail;
+
+		/* Mark that we need a new pointer to start aging... */
+		dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = INVALID_TAIL_PTR;
+		aging_tail = INVALID_TAIL_PTR;
+	}
+
+	/* Update the aging tail
+	 *
+	 * We throttle aging tail updates until we have a new tail that
+	 * represents >= one report more data than is already available for
+	 * reading. This ensures there will be enough data for a successful
+	 * read once this new pointer has aged and ensures we will give the new
+	 * pointer time to age.
+	 */
+	if (aging_tail == INVALID_TAIL_PTR &&
+	    (aged_tail == INVALID_TAIL_PTR ||
+	     OA_TAKEN(hw_tail, aged_tail) >= report_size)) {
+		struct i915_vma *vma = dev_priv->perf.oa.oa_buffer.vma;
+		u32 gtt_offset = i915_ggtt_offset(vma);
+
+		/* Be paranoid and do a bounds check on the pointer read back
+		 * from hardware, just in case some spurious hardware condition
+		 * could put the tail out of bounds...
+		 */
+		if (hw_tail >= gtt_offset &&
+		    hw_tail < (gtt_offset + OA_BUFFER_SIZE)) {
+			dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset =
+				aging_tail = hw_tail;
+			dev_priv->perf.oa.oa_buffer.aging_timestamp = now;
+		} else {
+			DRM_ERROR("Ignoring spurious out of range OA buffer tail pointer = %u\n",
+				  hw_tail);
+		}
+	}
+
+	spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	return aged_tail == INVALID_TAIL_PTR ?
+		false : OA_TAKEN(aged_tail, head) >= report_size;
 }
 
 /**
@@ -421,8 +575,6 @@ static int append_oa_sample(struct i915_perf_stream *stream,
  * @buf: destination buffer given by userspace
  * @count: the number of bytes userspace wants to read
  * @offset: (inout): the current position for writing into @buf
- * @head_ptr: (inout): the current oa buffer cpu read position
- * @tail: the current oa buffer gpu write position
  *
  * Notably any error condition resulting in a short read (-%ENOSPC or
  * -%EFAULT) will be returned even though one or more records may
@@ -431,66 +583,358 @@ static int append_oa_sample(struct i915_perf_stream *stream,
  * userspace.
  *
  * Note: reports are consumed from the head, and appended to the
- * tail, so the head chases the tail?... If you think that's mad
+ * tail, so the tail chases the head?... If you think that's mad
  * and back-to-front you're not alone, but this follows the
  * Gen PRM naming convention.
  *
  * Returns: 0 on success, negative error code on failure.
  */
-static int gen7_append_oa_reports(struct i915_perf_stream *stream,
+static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 				  char __user *buf,
 				  size_t count,
-				  size_t *offset,
-				  u32 *head_ptr,
-				  u32 tail)
+				  size_t *offset)
 {
 	struct drm_i915_private *dev_priv = stream->dev_priv;
 	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
 	u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr;
-	int tail_margin = dev_priv->perf.oa.tail_margin;
 	u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma);
 	u32 mask = (OA_BUFFER_SIZE - 1);
-	u32 head;
+	size_t start_offset = *offset;
+	unsigned long flags;
+	unsigned int aged_tail_idx;
+	u32 head, tail;
 	u32 taken;
 	int ret = 0;
 
 	if (WARN_ON(!stream->enabled))
 		return -EIO;
 
-	head = *head_ptr - gtt_offset;
+	spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	head = dev_priv->perf.oa.oa_buffer.head;
+	aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx;
+	tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset;
+
+	spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	/*
+	 * An invalid tail pointer here means we're still waiting for the poll
+	 * hrtimer callback to give us a pointer
+	 */
+	if (tail == INVALID_TAIL_PTR)
+		return -EAGAIN;
+
+	/*
+	 * NB: oa_buffer.head/tail include the gtt_offset which we don't want
+	 * while indexing relative to oa_buf_base.
+	 */
+	head -= gtt_offset;
 	tail -= gtt_offset;
 
-	/* The OA unit is expected to wrap the tail pointer according to the OA
-	 * buffer size and since we should never write a misaligned head
-	 * pointer we don't expect to read one back either...
+	/*
+	 * An out of bounds or misaligned head or tail pointer implies a driver
+	 * bug since we validate + align the tail pointers we read from the
+	 * hardware and we are in full control of the head pointer which should
+	 * only be incremented by multiples of the report size (notably also
+	 * all a power of two).
+	 */
+	if (WARN_ONCE(head > OA_BUFFER_SIZE || head % report_size ||
+		      tail > OA_BUFFER_SIZE || tail % report_size,
+		      "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
+		      head, tail))
+		return -EIO;
+
+
+	for (/* none */;
+	     (taken = OA_TAKEN(tail, head));
+	     head = (head + report_size) & mask) {
+		u8 *report = oa_buf_base + head;
+		u32 *report32 = (void *)report;
+		u32 ctx_id;
+		u32 reason;
+
+		/*
+		 * All the report sizes factor neatly into the buffer
+		 * size so we never expect to see a report split
+		 * between the beginning and end of the buffer.
+		 *
+		 * Given the initial alignment check a misalignment
+		 * here would imply a driver bug that would result
+		 * in an overrun.
+		 */
+		if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) {
+			DRM_ERROR("Spurious OA head ptr: non-integral report offset\n");
+			break;
+		}
+
+		/*
+		 * The reason field includes flags identifying what
+		 * triggered this specific report (mostly timer
+		 * triggered or e.g. due to a context switch).
+		 *
+		 * This field is never expected to be zero so we can
+		 * check that the report isn't invalid before copying
+		 * it to userspace...
+		 */
+		reason = ((report32[0] >> OAREPORT_REASON_SHIFT) &
+			  OAREPORT_REASON_MASK);
+		if (reason == 0) {
+			if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs))
+				DRM_NOTE("Skipping spurious, invalid OA report\n");
+			continue;
+		}
+
+		/*
+		 * XXX: Just keep the lower 21 bits for now since I'm not
+		 * entirely sure if the HW touches any of the higher bits in
+		 * this field
+		 */
+		ctx_id = report32[2] & 0x1fffff;
+
+		/*
+		 * Squash whatever is in the CTX_ID field if it's marked as
+		 * invalid to be sure we avoid false-positive, single-context
+		 * filtering below...
+		 *
+		 * Note: that we don't clear the valid_ctx_bit so userspace can
+		 * understand that the ID has been squashed by the kernel.
+		 */
+		if (!(report32[0] & dev_priv->perf.oa.gen8_valid_ctx_bit))
+			ctx_id = report32[2] = INVALID_CTX_ID;
+
+		/*
+		 * NB: For Gen 8 the OA unit no longer supports clock gating
+		 * off for a specific context and the kernel can't securely
+		 * stop the counters from updating as system-wide / global
+		 * values.
+		 *
+		 * Automatic reports now include a context ID so reports can be
+		 * filtered on the cpu but it's not worth trying to
+		 * automatically subtract/hide counter progress for other
+		 * contexts while filtering since we can't stop userspace
+		 * issuing MI_REPORT_PERF_COUNT commands which would still
+		 * provide a side-band view of the real values.
+		 *
+		 * To allow userspace (such as Mesa/GL_INTEL_performance_query)
+		 * to normalize counters for a single filtered context then it
+		 * needs be forwarded bookend context-switch reports so that it
+		 * can track switches in between MI_REPORT_PERF_COUNT commands
+		 * and can itself subtract/ignore the progress of counters
+		 * associated with other contexts. Note that the hardware
+		 * automatically triggers reports when switching to a new
+		 * context which are tagged with the ID of the newly active
+		 * context. To avoid the complexity (and likely fragility) of
+		 * reading ahead while parsing reports to try and minimize
+		 * forwarding redundant context switch reports (i.e. between
+		 * other, unrelated contexts) we simply elect to forward them
+		 * all.
+		 *
+		 * We don't rely solely on the reason field to identify context
+		 * switches since it's not-uncommon for periodic samples to
+		 * identify a switch before any 'context switch' report.
+		 */
+		if (!dev_priv->perf.oa.exclusive_stream->ctx ||
+		    dev_priv->perf.oa.specific_ctx_id == ctx_id ||
+		    (dev_priv->perf.oa.oa_buffer.last_ctx_id ==
+		     dev_priv->perf.oa.specific_ctx_id) ||
+		    reason & OAREPORT_REASON_CTX_SWITCH) {
+
+			/*
+			 * While filtering for a single context we avoid
+			 * leaking the IDs of other contexts.
+			 */
+			if (dev_priv->perf.oa.exclusive_stream->ctx &&
+			    dev_priv->perf.oa.specific_ctx_id != ctx_id) {
+				report32[2] = INVALID_CTX_ID;
+			}
+
+			ret = append_oa_sample(stream, buf, count, offset,
+					       report);
+			if (ret)
+				break;
+
+			dev_priv->perf.oa.oa_buffer.last_ctx_id = ctx_id;
+		}
+
+		/*
+		 * The above reason field sanity check is based on
+		 * the assumption that the OA buffer is initially
+		 * zeroed and we reset the field after copying so the
+		 * check is still meaningful once old reports start
+		 * being overwritten.
+		 */
+		report32[0] = 0;
+	}
+
+	if (start_offset != *offset) {
+		spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+		/*
+		 * We removed the gtt_offset for the copy loop above, indexing
+		 * relative to oa_buf_base so put back here...
+		 */
+		head += gtt_offset;
+
+		I915_WRITE(GEN8_OAHEADPTR, head & GEN8_OAHEADPTR_MASK);
+		dev_priv->perf.oa.oa_buffer.head = head;
+
+		spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+	}
+
+	return ret;
+}
+
+/**
+ * gen8_oa_read - copy status records then buffered OA reports
+ * @stream: An i915-perf stream opened for OA metrics
+ * @buf: destination buffer given by userspace
+ * @count: the number of bytes userspace wants to read
+ * @offset: (inout): the current position for writing into @buf
+ *
+ * Checks OA unit status registers and if necessary appends corresponding
+ * status records for userspace (such as for a buffer full condition) and then
+ * initiate appending any buffered OA reports.
+ *
+ * Updates @offset according to the number of bytes successfully copied into
+ * the userspace buffer.
+ *
+ * NB: some data may be successfully copied to the userspace buffer
+ * even if an error is returned, and this is reflected in the
+ * updated @offset.
+ *
+ * Returns: zero on success or a negative error code
+ */
+static int gen8_oa_read(struct i915_perf_stream *stream,
+			char __user *buf,
+			size_t count,
+			size_t *offset)
+{
+	struct drm_i915_private *dev_priv = stream->dev_priv;
+	u32 oastatus;
+	int ret;
+
+	if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr))
+		return -EIO;
+
+	oastatus = I915_READ(GEN8_OASTATUS);
+
+	/*
+	 * We treat OABUFFER_OVERFLOW as a significant error:
+	 *
+	 * Although theoretically we could handle this more gracefully
+	 * sometimes, some Gens don't correctly suppress certain
+	 * automatically triggered reports in this condition and so we
+	 * have to assume that old reports are now being trampled
+	 * over.
+	 *
+	 * Considering how we don't currently give userspace control
+	 * over the OA buffer size and always configure a large 16MB
+	 * buffer, then a buffer overflow does anyway likely indicate
+	 * that something has gone quite badly wrong.
 	 */
-	if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE ||
-	    head % report_size) {
-		DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n",
-			  head, tail);
+	if (oastatus & GEN8_OASTATUS_OABUFFER_OVERFLOW) {
+		ret = append_oa_status(stream, buf, count, offset,
+				       DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
+		if (ret)
+			return ret;
+
+		DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n",
+			  dev_priv->perf.oa.period_exponent);
+
 		dev_priv->perf.oa.ops.oa_disable(dev_priv);
 		dev_priv->perf.oa.ops.oa_enable(dev_priv);
-		*head_ptr = I915_READ(GEN7_OASTATUS2) &
-			GEN7_OASTATUS2_HEAD_MASK;
-		return -EIO;
+
+		/*
+		 * Note: .oa_enable() is expected to re-init the oabuffer and
+		 * reset GEN8_OASTATUS for us
+		 */
+		oastatus = I915_READ(GEN8_OASTATUS);
 	}
 
+	if (oastatus & GEN8_OASTATUS_REPORT_LOST) {
+		ret = append_oa_status(stream, buf, count, offset,
+				       DRM_I915_PERF_RECORD_OA_REPORT_LOST);
+		if (ret)
+			return ret;
+		I915_WRITE(GEN8_OASTATUS,
+			   oastatus & ~GEN8_OASTATUS_REPORT_LOST);
+	}
 
-	/* The tail pointer increases in 64 byte increments, not in report_size
-	 * steps...
+	return gen8_append_oa_reports(stream, buf, count, offset);
+}
+
+/**
+ * Copies all buffered OA reports into userspace read() buffer.
+ * @stream: An i915-perf stream opened for OA metrics
+ * @buf: destination buffer given by userspace
+ * @count: the number of bytes userspace wants to read
+ * @offset: (inout): the current position for writing into @buf
+ *
+ * Notably any error condition resulting in a short read (-%ENOSPC or
+ * -%EFAULT) will be returned even though one or more records may
+ * have been successfully copied. In this case it's up to the caller
+ * to decide if the error should be squashed before returning to
+ * userspace.
+ *
+ * Note: reports are consumed from the head, and appended to the
+ * tail, so the tail chases the head?... If you think that's mad
+ * and back-to-front you're not alone, but this follows the
+ * Gen PRM naming convention.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static int gen7_append_oa_reports(struct i915_perf_stream *stream,
+				  char __user *buf,
+				  size_t count,
+				  size_t *offset)
+{
+	struct drm_i915_private *dev_priv = stream->dev_priv;
+	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
+	u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr;
+	u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma);
+	u32 mask = (OA_BUFFER_SIZE - 1);
+	size_t start_offset = *offset;
+	unsigned long flags;
+	unsigned int aged_tail_idx;
+	u32 head, tail;
+	u32 taken;
+	int ret = 0;
+
+	if (WARN_ON(!stream->enabled))
+		return -EIO;
+
+	spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	head = dev_priv->perf.oa.oa_buffer.head;
+	aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx;
+	tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset;
+
+	spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	/* An invalid tail pointer here means we're still waiting for the poll
+	 * hrtimer callback to give us a pointer
 	 */
-	tail &= ~(report_size - 1);
+	if (tail == INVALID_TAIL_PTR)
+		return -EAGAIN;
 
-	/* Move the tail pointer back by the current tail_margin to account for
-	 * the possibility that the latest reports may not have really landed
-	 * in memory yet...
+	/* NB: oa_buffer.head/tail include the gtt_offset which we don't want
+	 * while indexing relative to oa_buf_base.
 	 */
+	head -= gtt_offset;
+	tail -= gtt_offset;
 
-	if (OA_TAKEN(tail, head) < report_size + tail_margin)
-		return -EAGAIN;
+	/* An out of bounds or misaligned head or tail pointer implies a driver
+	 * bug since we validate + align the tail pointers we read from the
+	 * hardware and we are in full control of the head pointer which should
+	 * only be incremented by multiples of the report size (notably also
+	 * all a power of two).
+	 */
+	if (WARN_ONCE(head > OA_BUFFER_SIZE || head % report_size ||
+		      tail > OA_BUFFER_SIZE || tail % report_size,
+		      "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
+		      head, tail))
+		return -EIO;
 
-	tail -= tail_margin;
-	tail &= mask;
 
 	for (/* none */;
 	     (taken = OA_TAKEN(tail, head));
@@ -518,7 +962,8 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream,
 		 * copying it to userspace...
 		 */
 		if (report32[0] == 0) {
-			DRM_NOTE("Skipping spurious, invalid OA report\n");
+			if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs))
+				DRM_NOTE("Skipping spurious, invalid OA report\n");
 			continue;
 		}
 
@@ -535,7 +980,21 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream,
 		report32[0] = 0;
 	}
 
-	*head_ptr = gtt_offset + head;
+	if (start_offset != *offset) {
+		spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+		/* We removed the gtt_offset for the copy loop above, indexing
+		 * relative to oa_buf_base so put back here...
+		 */
+		head += gtt_offset;
+
+		I915_WRITE(GEN7_OASTATUS2,
+			   ((head & GEN7_OASTATUS2_HEAD_MASK) |
+			    OA_MEM_SELECT_GGTT));
+		dev_priv->perf.oa.oa_buffer.head = head;
+
+		spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+	}
 
 	return ret;
 }
@@ -562,22 +1021,14 @@ static int gen7_oa_read(struct i915_perf_stream *stream,
 			size_t *offset)
 {
 	struct drm_i915_private *dev_priv = stream->dev_priv;
-	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
-	u32 oastatus2;
 	u32 oastatus1;
-	u32 head;
-	u32 tail;
 	int ret;
 
 	if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr))
 		return -EIO;
 
-	oastatus2 = I915_READ(GEN7_OASTATUS2);
 	oastatus1 = I915_READ(GEN7_OASTATUS1);
 
-	head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK;
-	tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
-
 	/* XXX: On Haswell we don't have a safe way to clear oastatus1
 	 * bits while the OA unit is enabled (while the tail pointer
 	 * may be updated asynchronously) so we ignore status bits
@@ -611,16 +1062,13 @@ static int gen7_oa_read(struct i915_perf_stream *stream,
 		if (ret)
 			return ret;
 
-		DRM_DEBUG("OA buffer overflow: force restart\n");
+		DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n",
+			  dev_priv->perf.oa.period_exponent);
 
 		dev_priv->perf.oa.ops.oa_disable(dev_priv);
 		dev_priv->perf.oa.ops.oa_enable(dev_priv);
 
-		oastatus2 = I915_READ(GEN7_OASTATUS2);
 		oastatus1 = I915_READ(GEN7_OASTATUS1);
-
-		head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK;
-		tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
 	}
 
 	if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) {
@@ -632,29 +1080,7 @@ static int gen7_oa_read(struct i915_perf_stream *stream,
 			GEN7_OASTATUS1_REPORT_LOST;
 	}
 
-	ret = gen7_append_oa_reports(stream, buf, count, offset,
-				     &head, tail);
-
-	/* All the report sizes are a power of two and the
-	 * head should always be incremented by some multiple
-	 * of the report size.
-	 *
-	 * A warning here, but notably if we later read back a
-	 * misaligned pointer we will treat that as a bug since
-	 * it could lead to a buffer overrun.
-	 */
-	WARN_ONCE(head & (report_size - 1),
-		  "i915: Writing misaligned OA head pointer");
-
-	/* Note: we update the head pointer here even if an error
-	 * was returned since the error may represent a short read
-	 * where some some reports were successfully copied.
-	 */
-	I915_WRITE(GEN7_OASTATUS2,
-		   ((head & GEN7_OASTATUS2_HEAD_MASK) |
-		    OA_MEM_SELECT_GGTT));
-
-	return ret;
+	return gen7_append_oa_reports(stream, buf, count, offset);
 }
 
 /**
@@ -679,14 +1105,8 @@ static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
 	if (!dev_priv->perf.oa.periodic)
 		return -EIO;
 
-	/* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it
-	 * just performs mmio reads of the OA buffer head + tail pointers and
-	 * it's assumed we're handling some operation that implies the stream
-	 * can't be destroyed until completion (such as a read()) that ensures
-	 * the device + OA buffer can't disappear
-	 */
 	return wait_event_interruptible(dev_priv->perf.oa.poll_wq,
-					!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv));
+					oa_buffer_check_unlocked(dev_priv));
 }
 
 /**
@@ -743,33 +1163,40 @@ static int i915_oa_read(struct i915_perf_stream *stream,
 static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
 {
 	struct drm_i915_private *dev_priv = stream->dev_priv;
-	struct intel_engine_cs *engine = dev_priv->engine[RCS];
-	int ret;
 
-	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
-	if (ret)
-		return ret;
+	if (i915.enable_execlists)
+		dev_priv->perf.oa.specific_ctx_id = stream->ctx->hw_id;
+	else {
+		struct intel_engine_cs *engine = dev_priv->engine[RCS];
+		struct intel_ring *ring;
+		int ret;
 
-	/* As the ID is the gtt offset of the context's vma we pin
-	 * the vma to ensure the ID remains fixed.
-	 *
-	 * NB: implied RCS engine...
-	 */
-	ret = engine->context_pin(engine, stream->ctx);
-	if (ret)
-		goto unlock;
+		ret = i915_mutex_lock_interruptible(&dev_priv->drm);
+		if (ret)
+			return ret;
 
-	/* Explicitly track the ID (instead of calling i915_ggtt_offset()
-	 * on the fly) considering the difference with gen8+ and
-	 * execlists
-	 */
-	dev_priv->perf.oa.specific_ctx_id =
-		i915_ggtt_offset(stream->ctx->engine[engine->id].state);
+		/*
+		 * As the ID is the gtt offset of the context's vma we
+		 * pin the vma to ensure the ID remains fixed.
+		 *
+		 * NB: implied RCS engine...
+		 */
+		ring = engine->context_pin(engine, stream->ctx);
+		mutex_unlock(&dev_priv->drm.struct_mutex);
+		if (IS_ERR(ring))
+			return PTR_ERR(ring);
 
-unlock:
-	mutex_unlock(&dev_priv->drm.struct_mutex);
 
-	return ret;
+		/*
+		 * Explicitly track the ID (instead of calling
+		 * i915_ggtt_offset() on the fly) considering the difference
+		 * with gen8+ and execlists
+		 */
+		dev_priv->perf.oa.specific_ctx_id =
+			i915_ggtt_offset(stream->ctx->engine[engine->id].state);
+	}
+
+	return 0;
 }
 
 /**
@@ -782,14 +1209,19 @@ unlock:
 static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
 {
 	struct drm_i915_private *dev_priv = stream->dev_priv;
-	struct intel_engine_cs *engine = dev_priv->engine[RCS];
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
+	if (i915.enable_execlists) {
+		dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
+	} else {
+		struct intel_engine_cs *engine = dev_priv->engine[RCS];
 
-	dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
-	engine->context_unpin(engine, stream->ctx);
+		mutex_lock(&dev_priv->drm.struct_mutex);
 
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+		dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
+		engine->context_unpin(engine, stream->ctx);
+
+		mutex_unlock(&dev_priv->drm.struct_mutex);
+	}
 }
 
 static void
@@ -813,6 +1245,12 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 
 	BUG_ON(stream != dev_priv->perf.oa.exclusive_stream);
 
+	/*
+	 * Unset exclusive_stream first, it might be checked while
+	 * disabling the metric set on gen8+.
+	 */
+	dev_priv->perf.oa.exclusive_stream = NULL;
+
 	dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
 
 	free_oa_buffer(dev_priv);
@@ -823,20 +1261,35 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 	if (stream->ctx)
 		oa_put_render_ctx_id(stream);
 
-	dev_priv->perf.oa.exclusive_stream = NULL;
+	if (dev_priv->perf.oa.spurious_report_rs.missed) {
+		DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n",
+			 dev_priv->perf.oa.spurious_report_rs.missed);
+	}
 }
 
 static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv)
 {
 	u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma);
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
 
 	/* Pre-DevBDW: OABUFFER must be set with counters off,
 	 * before OASTATUS1, but after OASTATUS2
 	 */
 	I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */
+	dev_priv->perf.oa.oa_buffer.head = gtt_offset;
+
 	I915_WRITE(GEN7_OABUFFER, gtt_offset);
+
 	I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */
 
+	/* Mark that we need updated tail pointers to read from... */
+	dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR;
+	dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR;
+
+	spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
 	/* On Haswell we have to track which OASTATUS1 flags we've
 	 * already seen since they can't be cleared while periodic
 	 * sampling is enabled.
@@ -862,6 +1315,65 @@ static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv)
 	dev_priv->perf.oa.pollin = false;
 }
 
+static void gen8_init_oa_buffer(struct drm_i915_private *dev_priv)
+{
+	u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma);
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	I915_WRITE(GEN8_OASTATUS, 0);
+	I915_WRITE(GEN8_OAHEADPTR, gtt_offset);
+	dev_priv->perf.oa.oa_buffer.head = gtt_offset;
+
+	I915_WRITE(GEN8_OABUFFER_UDW, 0);
+
+	/*
+	 * PRM says:
+	 *
+	 *  "This MMIO must be set before the OATAILPTR
+	 *  register and after the OAHEADPTR register. This is
+	 *  to enable proper functionality of the overflow
+	 *  bit."
+	 */
+	I915_WRITE(GEN8_OABUFFER, gtt_offset |
+		   OABUFFER_SIZE_16M | OA_MEM_SELECT_GGTT);
+	I915_WRITE(GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK);
+
+	/* Mark that we need updated tail pointers to read from... */
+	dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR;
+	dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR;
+
+	/*
+	 * Reset state used to recognise context switches, affecting which
+	 * reports we will forward to userspace while filtering for a single
+	 * context.
+	 */
+	dev_priv->perf.oa.oa_buffer.last_ctx_id = INVALID_CTX_ID;
+
+	spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
+
+	/*
+	 * NB: although the OA buffer will initially be allocated
+	 * zeroed via shmfs (and so this memset is redundant when
+	 * first allocating), we may re-init the OA buffer, either
+	 * when re-enabling a stream or in error/reset paths.
+	 *
+	 * The reason we clear the buffer for each re-init is for the
+	 * sanity check in gen8_append_oa_reports() that looks at the
+	 * reason field to make sure it's non-zero which relies on
+	 * the assumption that new reports are being written to zeroed
+	 * memory...
+	 */
+	memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
+
+	/*
+	 * Maybe make ->pollin per-stream state if we support multiple
+	 * concurrent streams in the future.
+	 */
+	dev_priv->perf.oa.pollin = false;
+}
+
 static int alloc_oa_buffer(struct drm_i915_private *dev_priv)
 {
 	struct drm_i915_gem_object *bo;
@@ -942,6 +1454,7 @@ static void config_oa_regs(struct drm_i915_private *dev_priv,
 static int hsw_enable_metric_set(struct drm_i915_private *dev_priv)
 {
 	int ret = i915_oa_select_metric_set_hsw(dev_priv);
+	int i;
 
 	if (ret)
 		return ret;
@@ -963,8 +1476,10 @@ static int hsw_enable_metric_set(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) |
 				  GEN6_CSUNIT_CLOCK_GATE_DISABLE));
 
-	config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs,
-		       dev_priv->perf.oa.mux_regs_len);
+	for (i = 0; i < dev_priv->perf.oa.n_mux_configs; i++) {
+		config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs[i],
+			       dev_priv->perf.oa.mux_regs_lens[i]);
+	}
 
 	/* It apparently takes a fairly long time for a new MUX
 	 * configuration to be be applied after these register writes.
@@ -1006,9 +1521,337 @@ static void hsw_disable_metric_set(struct drm_i915_private *dev_priv)
 				      ~GT_NOA_ENABLE));
 }
 
-static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv)
+/*
+ * NB: It must always remain pointer safe to run this even if the OA unit
+ * has been disabled.
+ *
+ * It's fine to put out-of-date values into these per-context registers
+ * in the case that the OA unit has been disabled.
+ */
+static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
+					   u32 *reg_state)
+{
+	struct drm_i915_private *dev_priv = ctx->i915;
+	const struct i915_oa_reg *flex_regs = dev_priv->perf.oa.flex_regs;
+	int n_flex_regs = dev_priv->perf.oa.flex_regs_len;
+	u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset;
+	u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset;
+	/* The MMIO offsets for Flex EU registers aren't contiguous */
+	u32 flex_mmio[] = {
+		i915_mmio_reg_offset(EU_PERF_CNTL0),
+		i915_mmio_reg_offset(EU_PERF_CNTL1),
+		i915_mmio_reg_offset(EU_PERF_CNTL2),
+		i915_mmio_reg_offset(EU_PERF_CNTL3),
+		i915_mmio_reg_offset(EU_PERF_CNTL4),
+		i915_mmio_reg_offset(EU_PERF_CNTL5),
+		i915_mmio_reg_offset(EU_PERF_CNTL6),
+	};
+	int i;
+
+	reg_state[ctx_oactxctrl] = i915_mmio_reg_offset(GEN8_OACTXCONTROL);
+	reg_state[ctx_oactxctrl+1] = (dev_priv->perf.oa.period_exponent <<
+				      GEN8_OA_TIMER_PERIOD_SHIFT) |
+				     (dev_priv->perf.oa.periodic ?
+				      GEN8_OA_TIMER_ENABLE : 0) |
+				     GEN8_OA_COUNTER_RESUME;
+
+	for (i = 0; i < ARRAY_SIZE(flex_mmio); i++) {
+		u32 state_offset = ctx_flexeu0 + i * 2;
+		u32 mmio = flex_mmio[i];
+
+		/*
+		 * This arbitrary default will select the 'EU FPU0 Pipeline
+		 * Active' event. In the future it's anticipated that there
+		 * will be an explicit 'No Event' we can select, but not yet...
+		 */
+		u32 value = 0;
+		int j;
+
+		for (j = 0; j < n_flex_regs; j++) {
+			if (i915_mmio_reg_offset(flex_regs[j].addr) == mmio) {
+				value = flex_regs[j].value;
+				break;
+			}
+		}
+
+		reg_state[state_offset] = mmio;
+		reg_state[state_offset+1] = value;
+	}
+}
+
+/*
+ * Same as gen8_update_reg_state_unlocked only through the batchbuffer. This
+ * is only used by the kernel context.
+ */
+static int gen8_emit_oa_config(struct drm_i915_gem_request *req)
+{
+	struct drm_i915_private *dev_priv = req->i915;
+	const struct i915_oa_reg *flex_regs = dev_priv->perf.oa.flex_regs;
+	int n_flex_regs = dev_priv->perf.oa.flex_regs_len;
+	/* The MMIO offsets for Flex EU registers aren't contiguous */
+	u32 flex_mmio[] = {
+		i915_mmio_reg_offset(EU_PERF_CNTL0),
+		i915_mmio_reg_offset(EU_PERF_CNTL1),
+		i915_mmio_reg_offset(EU_PERF_CNTL2),
+		i915_mmio_reg_offset(EU_PERF_CNTL3),
+		i915_mmio_reg_offset(EU_PERF_CNTL4),
+		i915_mmio_reg_offset(EU_PERF_CNTL5),
+		i915_mmio_reg_offset(EU_PERF_CNTL6),
+	};
+	u32 *cs;
+	int i;
+
+	cs = intel_ring_begin(req, n_flex_regs * 2 + 4);
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
+	*cs++ = MI_LOAD_REGISTER_IMM(n_flex_regs + 1);
+
+	*cs++ = i915_mmio_reg_offset(GEN8_OACTXCONTROL);
+	*cs++ = (dev_priv->perf.oa.period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
+		(dev_priv->perf.oa.periodic ? GEN8_OA_TIMER_ENABLE : 0) |
+		GEN8_OA_COUNTER_RESUME;
+
+	for (i = 0; i < ARRAY_SIZE(flex_mmio); i++) {
+		u32 mmio = flex_mmio[i];
+
+		/*
+		 * This arbitrary default will select the 'EU FPU0 Pipeline
+		 * Active' event. In the future it's anticipated that there
+		 * will be an explicit 'No Event' we can select, but not
+		 * yet...
+		 */
+		u32 value = 0;
+		int j;
+
+		for (j = 0; j < n_flex_regs; j++) {
+			if (i915_mmio_reg_offset(flex_regs[j].addr) == mmio) {
+				value = flex_regs[j].value;
+				break;
+			}
+		}
+
+		*cs++ = mmio;
+		*cs++ = value;
+	}
+
+	*cs++ = MI_NOOP;
+	intel_ring_advance(req, cs);
+
+	return 0;
+}
+
+static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine = dev_priv->engine[RCS];
+	struct i915_gem_timeline *timeline;
+	struct drm_i915_gem_request *req;
+	int ret;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+	i915_gem_retire_requests(dev_priv);
+
+	req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	ret = gen8_emit_oa_config(req);
+	if (ret) {
+		i915_add_request(req);
+		return ret;
+	}
+
+	/* Queue this switch after all other activity */
+	list_for_each_entry(timeline, &dev_priv->gt.timelines, link) {
+		struct drm_i915_gem_request *prev;
+		struct intel_timeline *tl;
+
+		tl = &timeline->engine[engine->id];
+		prev = i915_gem_active_raw(&tl->last_request,
+					   &dev_priv->drm.struct_mutex);
+		if (prev)
+			i915_sw_fence_await_sw_fence_gfp(&req->submit,
+							 &prev->submit,
+							 GFP_KERNEL);
+	}
+
+	ret = i915_switch_context(req);
+	i915_add_request(req);
+
+	return ret;
+}
+
+/*
+ * Manages updating the per-context aspects of the OA stream
+ * configuration across all contexts.
+ *
+ * The awkward consideration here is that OACTXCONTROL controls the
+ * exponent for periodic sampling which is primarily used for system
+ * wide profiling where we'd like a consistent sampling period even in
+ * the face of context switches.
+ *
+ * Our approach of updating the register state context (as opposed to
+ * say using a workaround batch buffer) ensures that the hardware
+ * won't automatically reload an out-of-date timer exponent even
+ * transiently before a WA BB could be parsed.
+ *
+ * This function needs to:
+ * - Ensure the currently running context's per-context OA state is
+ *   updated
+ * - Ensure that all existing contexts will have the correct per-context
+ *   OA state if they are scheduled for use.
+ * - Ensure any new contexts will be initialized with the correct
+ *   per-context OA state.
+ *
+ * Note: it's only the RCS/Render context that has any OA state.
+ */
+static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
+				       bool interruptible)
+{
+	struct i915_gem_context *ctx;
+	int ret;
+	unsigned int wait_flags = I915_WAIT_LOCKED;
+
+	if (interruptible) {
+		ret = i915_mutex_lock_interruptible(&dev_priv->drm);
+		if (ret)
+			return ret;
+
+		wait_flags |= I915_WAIT_INTERRUPTIBLE;
+	} else {
+		mutex_lock(&dev_priv->drm.struct_mutex);
+	}
+
+	/* Switch away from any user context. */
+	ret = gen8_switch_to_updated_kernel_context(dev_priv);
+	if (ret)
+		goto out;
+
+	/*
+	 * The OA register config is setup through the context image. This image
+	 * might be written to by the GPU on context switch (in particular on
+	 * lite-restore). This means we can't safely update a context's image,
+	 * if this context is scheduled/submitted to run on the GPU.
+	 *
+	 * We could emit the OA register config through the batch buffer but
+	 * this might leave small interval of time where the OA unit is
+	 * configured at an invalid sampling period.
+	 *
+	 * So far the best way to work around this issue seems to be draining
+	 * the GPU from any submitted work.
+	 */
+	ret = i915_gem_wait_for_idle(dev_priv, wait_flags);
+	if (ret)
+		goto out;
+
+	/* Update all contexts now that we've stalled the submission. */
+	list_for_each_entry(ctx, &dev_priv->context_list, link) {
+		struct intel_context *ce = &ctx->engine[RCS];
+		u32 *regs;
+
+		/* OA settings will be set upon first use */
+		if (!ce->state)
+			continue;
+
+		regs = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
+		if (IS_ERR(regs)) {
+			ret = PTR_ERR(regs);
+			goto out;
+		}
+
+		ce->state->obj->mm.dirty = true;
+		regs += LRC_STATE_PN * PAGE_SIZE / sizeof(*regs);
+
+		gen8_update_reg_state_unlocked(ctx, regs);
+
+		i915_gem_object_unpin_map(ce->state->obj);
+	}
+
+ out:
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	return ret;
+}
+
+static int gen8_enable_metric_set(struct drm_i915_private *dev_priv)
 {
-	lockdep_assert_held(&dev_priv->perf.hook_lock);
+	int ret = dev_priv->perf.oa.ops.select_metric_set(dev_priv);
+	int i;
+
+	if (ret)
+		return ret;
+
+	/*
+	 * We disable slice/unslice clock ratio change reports on SKL since
+	 * they are too noisy. The HW generates a lot of redundant reports
+	 * where the ratio hasn't really changed causing a lot of redundant
+	 * work to processes and increasing the chances we'll hit buffer
+	 * overruns.
+	 *
+	 * Although we don't currently use the 'disable overrun' OABUFFER
+	 * feature it's worth noting that clock ratio reports have to be
+	 * disabled before considering to use that feature since the HW doesn't
+	 * correctly block these reports.
+	 *
+	 * Currently none of the high-level metrics we have depend on knowing
+	 * this ratio to normalize.
+	 *
+	 * Note: This register is not power context saved and restored, but
+	 * that's OK considering that we disable RC6 while the OA unit is
+	 * enabled.
+	 *
+	 * The _INCLUDE_CLK_RATIO bit allows the slice/unslice frequency to
+	 * be read back from automatically triggered reports, as part of the
+	 * RPT_ID field.
+	 */
+	if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
+	    IS_KABYLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) {
+		I915_WRITE(GEN8_OA_DEBUG,
+			   _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
+					      GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
+	}
+
+	/*
+	 * Update all contexts prior writing the mux configurations as we need
+	 * to make sure all slices/subslices are ON before writing to NOA
+	 * registers.
+	 */
+	ret = gen8_configure_all_contexts(dev_priv, true);
+	if (ret)
+		return ret;
+
+	I915_WRITE(GDT_CHICKEN_BITS, 0xA0);
+	for (i = 0; i < dev_priv->perf.oa.n_mux_configs; i++) {
+		config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs[i],
+			       dev_priv->perf.oa.mux_regs_lens[i]);
+	}
+	I915_WRITE(GDT_CHICKEN_BITS, 0x80);
+
+	config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs,
+		       dev_priv->perf.oa.b_counter_regs_len);
+
+	return 0;
+}
+
+static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
+{
+	/* Reset all contexts' slices/subslices configurations. */
+	gen8_configure_all_contexts(dev_priv, false);
+}
+
+static void gen7_oa_enable(struct drm_i915_private *dev_priv)
+{
+	/*
+	 * Reset buf pointers so we don't forward reports from before now.
+	 *
+	 * Think carefully if considering trying to avoid this, since it
+	 * also ensures status flags and the buffer itself are cleared
+	 * in error paths, and we have checks for invalid reports based
+	 * on the assumption that certain fields are written to zeroed
+	 * memory which this helps maintains.
+	 */
+	gen7_init_oa_buffer(dev_priv);
 
 	if (dev_priv->perf.oa.exclusive_stream->enabled) {
 		struct i915_gem_context *ctx =
@@ -1031,11 +1874,12 @@ static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv)
 		I915_WRITE(GEN7_OACONTROL, 0);
 }
 
-static void gen7_oa_enable(struct drm_i915_private *dev_priv)
+static void gen8_oa_enable(struct drm_i915_private *dev_priv)
 {
-	unsigned long flags;
+	u32 report_format = dev_priv->perf.oa.oa_buffer.format;
 
-	/* Reset buf pointers so we don't forward reports from before now.
+	/*
+	 * Reset buf pointers so we don't forward reports from before now.
 	 *
 	 * Think carefully if considering trying to avoid this, since it
 	 * also ensures status flags and the buffer itself are cleared
@@ -1043,11 +1887,16 @@ static void gen7_oa_enable(struct drm_i915_private *dev_priv)
 	 * on the assumption that certain fields are written to zeroed
 	 * memory which this helps maintains.
 	 */
-	gen7_init_oa_buffer(dev_priv);
+	gen8_init_oa_buffer(dev_priv);
 
-	spin_lock_irqsave(&dev_priv->perf.hook_lock, flags);
-	gen7_update_oacontrol_locked(dev_priv);
-	spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags);
+	/*
+	 * Note: we don't rely on the hardware to perform single context
+	 * filtering and instead filter on the cpu based on the context-id
+	 * field of reports
+	 */
+	I915_WRITE(GEN8_OACONTROL, (report_format <<
+				    GEN8_OA_REPORT_FORMAT_SHIFT) |
+				   GEN8_OA_COUNTER_ENABLE);
 }
 
 /**
@@ -1076,6 +1925,11 @@ static void gen7_oa_disable(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN7_OACONTROL, 0);
 }
 
+static void gen8_oa_disable(struct drm_i915_private *dev_priv)
+{
+	I915_WRITE(GEN8_OACONTROL, 0);
+}
+
 /**
  * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
  * @stream: An i915 perf stream opened for OA metrics
@@ -1094,12 +1948,6 @@ static void i915_oa_stream_disable(struct i915_perf_stream *stream)
 		hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer);
 }
 
-static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent)
-{
-	return div_u64(1000000000ULL * (2ULL << exponent),
-		       dev_priv->perf.oa.timestamp_frequency);
-}
-
 static const struct i915_perf_stream_ops i915_oa_stream_ops = {
 	.destroy = i915_oa_stream_destroy,
 	.enable = i915_oa_stream_enable,
@@ -1173,6 +2021,26 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 		return -EINVAL;
 	}
 
+	/* We set up some ratelimit state to potentially throttle any _NOTES
+	 * about spurious, invalid OA reports which we don't forward to
+	 * userspace.
+	 *
+	 * The initialization is associated with opening the stream (not driver
+	 * init) considering we print a _NOTE about any throttling when closing
+	 * the stream instead of waiting until driver _fini which no one would
+	 * ever see.
+	 *
+	 * Using the same limiting factors as printk_ratelimit()
+	 */
+	ratelimit_state_init(&dev_priv->perf.oa.spurious_report_rs,
+			     5 * HZ, 10);
+	/* Since we use a DRM_NOTE for spurious reports it would be
+	 * inconsistent to let __ratelimit() automatically print a warning for
+	 * throttling.
+	 */
+	ratelimit_set_flags(&dev_priv->perf.oa.spurious_report_rs,
+			    RATELIMIT_MSG_ON_RELEASE);
+
 	stream->sample_size = sizeof(struct drm_i915_perf_record_header);
 
 	format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size;
@@ -1190,20 +2058,9 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	dev_priv->perf.oa.metrics_set = props->metrics_set;
 
 	dev_priv->perf.oa.periodic = props->oa_periodic;
-	if (dev_priv->perf.oa.periodic) {
-		u32 tail;
-
+	if (dev_priv->perf.oa.periodic)
 		dev_priv->perf.oa.period_exponent = props->oa_period_exponent;
 
-		/* See comment for OA_TAIL_MARGIN_NSEC for details
-		 * about this tail_margin...
-		 */
-		tail = div64_u64(OA_TAIL_MARGIN_NSEC,
-				 oa_exponent_to_ns(dev_priv,
-						   props->oa_period_exponent));
-		dev_priv->perf.oa.tail_margin = (tail + 1) * format_size;
-	}
-
 	if (stream->ctx) {
 		ret = oa_get_render_ctx_id(stream);
 		if (ret)
@@ -1251,6 +2108,21 @@ err_oa_buf_alloc:
 	return ret;
 }
 
+void i915_oa_init_reg_state(struct intel_engine_cs *engine,
+			    struct i915_gem_context *ctx,
+			    u32 *reg_state)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+
+	if (engine->id != RCS)
+		return;
+
+	if (!dev_priv->perf.initialized)
+		return;
+
+	gen8_update_reg_state_unlocked(ctx, reg_state);
+}
+
 /**
  * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation
  * @stream: An i915 perf stream
@@ -1352,7 +2224,15 @@ static ssize_t i915_perf_read(struct file *file,
 		mutex_unlock(&dev_priv->perf.lock);
 	}
 
-	if (ret >= 0) {
+	/* We allow the poll checking to sometimes report false positive POLLIN
+	 * events where we might actually report EAGAIN on read() if there's
+	 * not really any data available. In this situation though we don't
+	 * want to enter a busy loop between poll() reporting a POLLIN event
+	 * and read() returning -EAGAIN. Clearing the oa.pollin state here
+	 * effectively ensures we back off until the next hrtimer callback
+	 * before reporting another POLLIN event.
+	 */
+	if (ret >= 0 || ret == -EAGAIN) {
 		/* Maybe make ->pollin per-stream state if we support multiple
 		 * concurrent streams in the future.
 		 */
@@ -1368,7 +2248,7 @@ static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer)
 		container_of(hrtimer, typeof(*dev_priv),
 			     perf.oa.poll_check_timer);
 
-	if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) {
+	if (oa_buffer_check_unlocked(dev_priv)) {
 		dev_priv->perf.oa.pollin = true;
 		wake_up(&dev_priv->perf.oa.poll_wq);
 	}
@@ -1657,6 +2537,7 @@ i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv,
 	struct i915_gem_context *specific_ctx = NULL;
 	struct i915_perf_stream *stream = NULL;
 	unsigned long f_flags = 0;
+	bool privileged_op = true;
 	int stream_fd;
 	int ret;
 
@@ -1674,12 +2555,29 @@ i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv,
 		}
 	}
 
+	/*
+	 * On Haswell the OA unit supports clock gating off for a specific
+	 * context and in this mode there's no visibility of metrics for the
+	 * rest of the system, which we consider acceptable for a
+	 * non-privileged client.
+	 *
+	 * For Gen8+ the OA unit no longer supports clock gating off for a
+	 * specific context and the kernel can't securely stop the counters
+	 * from updating as system-wide / global values. Even though we can
+	 * filter reports based on the included context ID we can't block
+	 * clients from seeing the raw / global counter values via
+	 * MI_REPORT_PERF_COUNT commands and so consider it a privileged op to
+	 * enable the OA unit by default.
+	 */
+	if (IS_HASWELL(dev_priv) && specific_ctx)
+		privileged_op = false;
+
 	/* Similar to perf's kernel.perf_paranoid_cpu sysctl option
 	 * we check a dev.i915.perf_stream_paranoid sysctl option
 	 * to determine if it's ok to access system wide OA counters
 	 * without CAP_SYS_ADMIN privileges.
 	 */
-	if (!specific_ctx &&
+	if (privileged_op &&
 	    i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) {
 		DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n");
 		ret = -EACCES;
@@ -1740,6 +2638,12 @@ err:
 	return ret;
 }
 
+static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent)
+{
+	return div_u64(1000000000ULL * (2ULL << exponent),
+		       dev_priv->perf.oa.timestamp_frequency);
+}
+
 /**
  * read_properties_unlocked - validate + copy userspace stream open properties
  * @dev_priv: i915 device instance
@@ -1817,11 +2721,13 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv,
 			break;
 		case DRM_I915_PERF_PROP_OA_FORMAT:
 			if (value == 0 || value >= I915_OA_FORMAT_MAX) {
-				DRM_DEBUG("Invalid OA report format\n");
+				DRM_DEBUG("Out-of-range OA report format %llu\n",
+					  value);
 				return -EINVAL;
 			}
 			if (!dev_priv->perf.oa.oa_formats[value].size) {
-				DRM_DEBUG("Invalid OA report format\n");
+				DRM_DEBUG("Unsupported OA report format %llu\n",
+					  value);
 				return -EINVAL;
 			}
 			props->oa_format = value;
@@ -1834,16 +2740,13 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv,
 			}
 
 			/* Theoretically we can program the OA unit to sample
-			 * every 160ns but don't allow that by default unless
-			 * root.
-			 *
-			 * On Haswell the period is derived from the exponent
-			 * as:
-			 *
-			 *   period = 80ns * 2^(exponent + 1)
+			 * e.g. every 160ns for HSW, 167ns for BDW/SKL or 104ns
+			 * for BXT. We don't allow such high sampling
+			 * frequencies by default unless root.
 			 */
+
 			BUILD_BUG_ON(sizeof(oa_period) != 8);
-			oa_period = 80ull * (2ull << value);
+			oa_period = oa_exponent_to_ns(dev_priv, value);
 
 			/* This check is primarily to ensure that oa_period <=
 			 * UINT32_MAX (before passing to do_div which only
@@ -1949,9 +2852,6 @@ int i915_perf_open_ioctl(struct drm_device *dev, void *data,
  */
 void i915_perf_register(struct drm_i915_private *dev_priv)
 {
-	if (!IS_HASWELL(dev_priv))
-		return;
-
 	if (!dev_priv->perf.initialized)
 		return;
 
@@ -1967,11 +2867,50 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
 	if (!dev_priv->perf.metrics_kobj)
 		goto exit;
 
-	if (i915_perf_register_sysfs_hsw(dev_priv)) {
-		kobject_put(dev_priv->perf.metrics_kobj);
-		dev_priv->perf.metrics_kobj = NULL;
+	if (IS_HASWELL(dev_priv)) {
+		if (i915_perf_register_sysfs_hsw(dev_priv))
+			goto sysfs_error;
+	} else if (IS_BROADWELL(dev_priv)) {
+		if (i915_perf_register_sysfs_bdw(dev_priv))
+			goto sysfs_error;
+	} else if (IS_CHERRYVIEW(dev_priv)) {
+		if (i915_perf_register_sysfs_chv(dev_priv))
+			goto sysfs_error;
+	} else if (IS_SKYLAKE(dev_priv)) {
+		if (IS_SKL_GT2(dev_priv)) {
+			if (i915_perf_register_sysfs_sklgt2(dev_priv))
+				goto sysfs_error;
+		} else if (IS_SKL_GT3(dev_priv)) {
+			if (i915_perf_register_sysfs_sklgt3(dev_priv))
+				goto sysfs_error;
+		} else if (IS_SKL_GT4(dev_priv)) {
+			if (i915_perf_register_sysfs_sklgt4(dev_priv))
+				goto sysfs_error;
+		} else
+			goto sysfs_error;
+	} else if (IS_BROXTON(dev_priv)) {
+		if (i915_perf_register_sysfs_bxt(dev_priv))
+			goto sysfs_error;
+	} else if (IS_KABYLAKE(dev_priv)) {
+		if (IS_KBL_GT2(dev_priv)) {
+			if (i915_perf_register_sysfs_kblgt2(dev_priv))
+				goto sysfs_error;
+		} else if (IS_KBL_GT3(dev_priv)) {
+			if (i915_perf_register_sysfs_kblgt3(dev_priv))
+				goto sysfs_error;
+		} else
+			goto sysfs_error;
+	} else if (IS_GEMINILAKE(dev_priv)) {
+		if (i915_perf_register_sysfs_glk(dev_priv))
+			goto sysfs_error;
 	}
 
+	goto exit;
+
+sysfs_error:
+	kobject_put(dev_priv->perf.metrics_kobj);
+	dev_priv->perf.metrics_kobj = NULL;
+
 exit:
 	mutex_unlock(&dev_priv->perf.lock);
 }
@@ -1987,13 +2926,32 @@ exit:
  */
 void i915_perf_unregister(struct drm_i915_private *dev_priv)
 {
-	if (!IS_HASWELL(dev_priv))
-		return;
-
 	if (!dev_priv->perf.metrics_kobj)
 		return;
 
-	i915_perf_unregister_sysfs_hsw(dev_priv);
+	if (IS_HASWELL(dev_priv))
+		i915_perf_unregister_sysfs_hsw(dev_priv);
+	else if (IS_BROADWELL(dev_priv))
+		i915_perf_unregister_sysfs_bdw(dev_priv);
+	else if (IS_CHERRYVIEW(dev_priv))
+		i915_perf_unregister_sysfs_chv(dev_priv);
+	else if (IS_SKYLAKE(dev_priv)) {
+		if (IS_SKL_GT2(dev_priv))
+			i915_perf_unregister_sysfs_sklgt2(dev_priv);
+		else if (IS_SKL_GT3(dev_priv))
+			i915_perf_unregister_sysfs_sklgt3(dev_priv);
+		else if (IS_SKL_GT4(dev_priv))
+			i915_perf_unregister_sysfs_sklgt4(dev_priv);
+	} else if (IS_BROXTON(dev_priv))
+		i915_perf_unregister_sysfs_bxt(dev_priv);
+	else if (IS_KABYLAKE(dev_priv)) {
+		if (IS_KBL_GT2(dev_priv))
+			i915_perf_unregister_sysfs_kblgt2(dev_priv);
+		else if (IS_KBL_GT3(dev_priv))
+			i915_perf_unregister_sysfs_kblgt3(dev_priv);
+	} else if (IS_GEMINILAKE(dev_priv))
+		i915_perf_unregister_sysfs_glk(dev_priv);
+
 
 	kobject_put(dev_priv->perf.metrics_kobj);
 	dev_priv->perf.metrics_kobj = NULL;
@@ -2052,37 +3010,133 @@ static struct ctl_table dev_root[] = {
  */
 void i915_perf_init(struct drm_i915_private *dev_priv)
 {
-	if (!IS_HASWELL(dev_priv))
-		return;
+	dev_priv->perf.oa.n_builtin_sets = 0;
+
+	if (IS_HASWELL(dev_priv)) {
+		dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer;
+		dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set;
+		dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set;
+		dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable;
+		dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable;
+		dev_priv->perf.oa.ops.read = gen7_oa_read;
+		dev_priv->perf.oa.ops.oa_hw_tail_read =
+			gen7_oa_hw_tail_read;
+
+		dev_priv->perf.oa.timestamp_frequency = 12500000;
+
+		dev_priv->perf.oa.oa_formats = hsw_oa_formats;
+
+		dev_priv->perf.oa.n_builtin_sets =
+			i915_oa_n_builtin_metric_sets_hsw;
+	} else if (i915.enable_execlists) {
+		/* Note: that although we could theoretically also support the
+		 * legacy ringbuffer mode on BDW (and earlier iterations of
+		 * this driver, before upstreaming did this) it didn't seem
+		 * worth the complexity to maintain now that BDW+ enable
+		 * execlist mode by default.
+		 */
+
+		if (IS_GEN8(dev_priv)) {
+			dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120;
+			dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce;
 
-	hrtimer_init(&dev_priv->perf.oa.poll_check_timer,
-		     CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-	dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb;
-	init_waitqueue_head(&dev_priv->perf.oa.poll_wq);
+			dev_priv->perf.oa.timestamp_frequency = 12500000;
 
-	INIT_LIST_HEAD(&dev_priv->perf.streams);
-	mutex_init(&dev_priv->perf.lock);
-	spin_lock_init(&dev_priv->perf.hook_lock);
+			dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25);
 
-	dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer;
-	dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set;
-	dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set;
-	dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable;
-	dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable;
-	dev_priv->perf.oa.ops.read = gen7_oa_read;
-	dev_priv->perf.oa.ops.oa_buffer_is_empty =
-		gen7_oa_buffer_is_empty_fop_unlocked;
+			if (IS_BROADWELL(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_bdw;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_bdw;
+			} else if (IS_CHERRYVIEW(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_chv;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_chv;
+			}
+		} else if (IS_GEN9(dev_priv)) {
+			dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
+			dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
+
+			dev_priv->perf.oa.timestamp_frequency = 12000000;
+
+			dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
+
+			if (IS_SKL_GT2(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_sklgt2;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_sklgt2;
+			} else if (IS_SKL_GT3(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_sklgt3;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_sklgt3;
+			} else if (IS_SKL_GT4(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_sklgt4;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_sklgt4;
+			} else if (IS_BROXTON(dev_priv)) {
+				dev_priv->perf.oa.timestamp_frequency = 19200000;
+
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_bxt;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_bxt;
+			} else if (IS_KBL_GT2(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_kblgt2;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_kblgt2;
+			} else if (IS_KBL_GT3(dev_priv)) {
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_kblgt3;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_kblgt3;
+			} else if (IS_GEMINILAKE(dev_priv)) {
+				dev_priv->perf.oa.timestamp_frequency = 19200000;
+
+				dev_priv->perf.oa.n_builtin_sets =
+					i915_oa_n_builtin_metric_sets_glk;
+				dev_priv->perf.oa.ops.select_metric_set =
+					i915_oa_select_metric_set_glk;
+			}
+		}
 
-	dev_priv->perf.oa.timestamp_frequency = 12500000;
+		if (dev_priv->perf.oa.n_builtin_sets) {
+			dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer;
+			dev_priv->perf.oa.ops.enable_metric_set =
+				gen8_enable_metric_set;
+			dev_priv->perf.oa.ops.disable_metric_set =
+				gen8_disable_metric_set;
+			dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable;
+			dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable;
+			dev_priv->perf.oa.ops.read = gen8_oa_read;
+			dev_priv->perf.oa.ops.oa_hw_tail_read =
+				gen8_oa_hw_tail_read;
+
+			dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
+		}
+	}
 
-	dev_priv->perf.oa.oa_formats = hsw_oa_formats;
+	if (dev_priv->perf.oa.n_builtin_sets) {
+		hrtimer_init(&dev_priv->perf.oa.poll_check_timer,
+				CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+		dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb;
+		init_waitqueue_head(&dev_priv->perf.oa.poll_wq);
 
-	dev_priv->perf.oa.n_builtin_sets =
-		i915_oa_n_builtin_metric_sets_hsw;
+		INIT_LIST_HEAD(&dev_priv->perf.streams);
+		mutex_init(&dev_priv->perf.lock);
+		spin_lock_init(&dev_priv->perf.oa.oa_buffer.ptr_lock);
 
-	dev_priv->perf.sysctl_header = register_sysctl_table(dev_root);
+		oa_sample_rate_hard_limit =
+			dev_priv->perf.oa.timestamp_frequency / 2;
+		dev_priv->perf.sysctl_header = register_sysctl_table(dev_root);
 
-	dev_priv->perf.initialized = true;
+		dev_priv->perf.initialized = true;
+	}
 }
 
 /**
@@ -2097,5 +3151,6 @@ void i915_perf_fini(struct drm_i915_private *dev_priv)
 	unregister_sysctl_table(dev_priv->perf.sysctl_header);
 
 	memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops));
+
 	dev_priv->perf.initialized = false;
 }
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 65b837e96fe6..c8647cfa81ba 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -58,10 +58,13 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b))
 #define _PORT(port, a, b) ((a) + (port)*((b)-(a)))
 #define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b))
-#define _PIPE3(pipe, ...) _PICK(pipe, __VA_ARGS__)
-#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PIPE3(pipe, a, b, c))
-#define _PORT3(port, ...) _PICK(port, __VA_ARGS__)
-#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PORT3(pipe, a, b, c))
+#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
+#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
+#define _PLL(pll, a, b) ((a) + (pll)*((b)-(a)))
+#define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b))
+#define _MMIO_PORT6(port, a, b, c, d, e, f) _MMIO(_PICK(port, a, b, c, d, e, f))
+#define _MMIO_PORT6_LN(port, ln, a0, a1, b, c, d, e, f)			\
+	_MMIO(_PICK(port, a0, b, c, d, e, f) + (ln * (a1 - a0)))
 #define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__)
 #define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c))
 
@@ -85,6 +88,14 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define VECS_HW		3
 #define VCS2_HW		4
 
+/* Engine class */
+
+#define RENDER_CLASS		0
+#define VIDEO_DECODE_CLASS	1
+#define VIDEO_ENHANCEMENT_CLASS	2
+#define COPY_ENGINE_CLASS	3
+#define OTHER_CLASS		4
+
 /* PCI config space */
 
 #define MCHBAR_I915 0x44
@@ -645,6 +656,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 
 #define GEN8_OACTXID _MMIO(0x2364)
 
+#define GEN8_OA_DEBUG _MMIO(0x2B04)
+#define  GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS    (1<<5)
+#define  GEN9_OA_DEBUG_INCLUDE_CLK_RATIO	    (1<<6)
+#define  GEN9_OA_DEBUG_DISABLE_GO_1_0_REPORTS	    (1<<2)
+#define  GEN9_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS   (1<<1)
+
 #define GEN8_OACONTROL _MMIO(0x2B00)
 #define  GEN8_OA_REPORT_FORMAT_A12	    (0<<2)
 #define  GEN8_OA_REPORT_FORMAT_A12_B8_C8    (2<<2)
@@ -666,6 +683,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN7_OABUFFER_STOP_RESUME_ENABLE   (1<<1)
 #define  GEN7_OABUFFER_RESUME		    (1<<0)
 
+#define GEN8_OABUFFER_UDW _MMIO(0x23b4)
 #define GEN8_OABUFFER _MMIO(0x2b14)
 
 #define GEN7_OASTATUS1 _MMIO(0x2364)
@@ -684,7 +702,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN8_OASTATUS_REPORT_LOST	    (1<<0)
 
 #define GEN8_OAHEADPTR _MMIO(0x2B0C)
+#define GEN8_OAHEADPTR_MASK    0xffffffc0
 #define GEN8_OATAILPTR _MMIO(0x2B10)
+#define GEN8_OATAILPTR_MASK    0xffffffc0
 
 #define OABUFFER_SIZE_128K  (0<<3)
 #define OABUFFER_SIZE_256K  (1<<3)
@@ -697,7 +717,17 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 
 #define OA_MEM_SELECT_GGTT  (1<<0)
 
+/*
+ * Flexible, Aggregate EU Counter Registers.
+ * Note: these aren't contiguous
+ */
 #define EU_PERF_CNTL0	    _MMIO(0xe458)
+#define EU_PERF_CNTL1	    _MMIO(0xe558)
+#define EU_PERF_CNTL2	    _MMIO(0xe658)
+#define EU_PERF_CNTL3	    _MMIO(0xe758)
+#define EU_PERF_CNTL4	    _MMIO(0xe45c)
+#define EU_PERF_CNTL5	    _MMIO(0xe55c)
+#define EU_PERF_CNTL6	    _MMIO(0xe65c)
 
 #define GDT_CHICKEN_BITS    _MMIO(0x9840)
 #define GT_NOA_ENABLE	    0x00000080
@@ -1057,6 +1087,7 @@ enum skl_disp_power_wells {
 	SKL_DISP_PW_MISC_IO,
 	SKL_DISP_PW_DDI_A_E,
 	GLK_DISP_PW_DDI_A = SKL_DISP_PW_DDI_A_E,
+	CNL_DISP_PW_DDI_A = SKL_DISP_PW_DDI_A_E,
 	SKL_DISP_PW_DDI_B,
 	SKL_DISP_PW_DDI_C,
 	SKL_DISP_PW_DDI_D,
@@ -1064,6 +1095,10 @@ enum skl_disp_power_wells {
 	GLK_DISP_PW_AUX_A = 8,
 	GLK_DISP_PW_AUX_B,
 	GLK_DISP_PW_AUX_C,
+	CNL_DISP_PW_AUX_A = GLK_DISP_PW_AUX_A,
+	CNL_DISP_PW_AUX_B = GLK_DISP_PW_AUX_B,
+	CNL_DISP_PW_AUX_C = GLK_DISP_PW_AUX_C,
+	CNL_DISP_PW_AUX_D,
 
 	SKL_DISP_PW_1 = 14,
 	SKL_DISP_PW_2,
@@ -1650,6 +1685,10 @@ enum skl_disp_power_wells {
 #define   PHY_RESERVED			(1 << 7)
 #define BXT_PORT_CL1CM_DW0(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW0_BC)
 
+#define CNL_PORT_CL1CM_DW5		_MMIO(0x162014)
+#define   CL_POWER_DOWN_ENABLE		(1 << 4)
+#define   SUS_CLOCK_CONFIG		(3 << 0)
+
 #define _PORT_CL1CM_DW9_A		0x162024
 #define _PORT_CL1CM_DW9_BC		0x6C024
 #define   IREF0RC_OFFSET_SHIFT		8
@@ -1674,6 +1713,155 @@ enum skl_disp_power_wells {
 #define   OCL2_LDOFUSE_PWR_DIS		(1 << 6)
 #define BXT_PORT_CL1CM_DW30(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW30_BC)
 
+#define _CNL_PORT_PCS_DW1_GRP_AE	0x162304
+#define _CNL_PORT_PCS_DW1_GRP_B		0x162384
+#define _CNL_PORT_PCS_DW1_GRP_C		0x162B04
+#define _CNL_PORT_PCS_DW1_GRP_D		0x162B84
+#define _CNL_PORT_PCS_DW1_GRP_F		0x162A04
+#define _CNL_PORT_PCS_DW1_LN0_AE	0x162404
+#define _CNL_PORT_PCS_DW1_LN0_B		0x162604
+#define _CNL_PORT_PCS_DW1_LN0_C		0x162C04
+#define _CNL_PORT_PCS_DW1_LN0_D		0x162E04
+#define _CNL_PORT_PCS_DW1_LN0_F		0x162804
+#define CNL_PORT_PCS_DW1_GRP(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_PCS_DW1_GRP_AE, \
+						    _CNL_PORT_PCS_DW1_GRP_B, \
+						    _CNL_PORT_PCS_DW1_GRP_C, \
+						    _CNL_PORT_PCS_DW1_GRP_D, \
+						    _CNL_PORT_PCS_DW1_GRP_AE, \
+						    _CNL_PORT_PCS_DW1_GRP_F)
+#define CNL_PORT_PCS_DW1_LN0(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_PCS_DW1_LN0_AE, \
+						    _CNL_PORT_PCS_DW1_LN0_B, \
+						    _CNL_PORT_PCS_DW1_LN0_C, \
+						    _CNL_PORT_PCS_DW1_LN0_D, \
+						    _CNL_PORT_PCS_DW1_LN0_AE, \
+						    _CNL_PORT_PCS_DW1_LN0_F)
+#define   COMMON_KEEPER_EN		(1 << 26)
+
+#define _CNL_PORT_TX_DW2_GRP_AE		0x162348
+#define _CNL_PORT_TX_DW2_GRP_B		0x1623C8
+#define _CNL_PORT_TX_DW2_GRP_C		0x162B48
+#define _CNL_PORT_TX_DW2_GRP_D		0x162BC8
+#define _CNL_PORT_TX_DW2_GRP_F		0x162A48
+#define _CNL_PORT_TX_DW2_LN0_AE		0x162448
+#define _CNL_PORT_TX_DW2_LN0_B		0x162648
+#define _CNL_PORT_TX_DW2_LN0_C		0x162C48
+#define _CNL_PORT_TX_DW2_LN0_D		0x162E48
+#define _CNL_PORT_TX_DW2_LN0_F		0x162A48
+#define CNL_PORT_TX_DW2_GRP(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW2_GRP_AE, \
+						    _CNL_PORT_TX_DW2_GRP_B, \
+						    _CNL_PORT_TX_DW2_GRP_C, \
+						    _CNL_PORT_TX_DW2_GRP_D, \
+						    _CNL_PORT_TX_DW2_GRP_AE, \
+						    _CNL_PORT_TX_DW2_GRP_F)
+#define CNL_PORT_TX_DW2_LN0(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW2_LN0_AE, \
+						    _CNL_PORT_TX_DW2_LN0_B, \
+						    _CNL_PORT_TX_DW2_LN0_C, \
+						    _CNL_PORT_TX_DW2_LN0_D, \
+						    _CNL_PORT_TX_DW2_LN0_AE, \
+						    _CNL_PORT_TX_DW2_LN0_F)
+#define   SWING_SEL_UPPER(x)		((x >> 3) << 15)
+#define   SWING_SEL_UPPER_MASK		(1 << 15)
+#define   SWING_SEL_LOWER(x)		((x & 0x7) << 11)
+#define   SWING_SEL_LOWER_MASK		(0x7 << 11)
+#define   RCOMP_SCALAR(x)		((x) << 0)
+#define   RCOMP_SCALAR_MASK		(0xFF << 0)
+
+#define _CNL_PORT_TX_DW4_GRP_AE		0x162350
+#define _CNL_PORT_TX_DW4_GRP_B		0x1623D0
+#define _CNL_PORT_TX_DW4_GRP_C		0x162B50
+#define _CNL_PORT_TX_DW4_GRP_D		0x162BD0
+#define _CNL_PORT_TX_DW4_GRP_F		0x162A50
+#define _CNL_PORT_TX_DW4_LN0_AE		0x162450
+#define _CNL_PORT_TX_DW4_LN1_AE		0x1624D0
+#define _CNL_PORT_TX_DW4_LN0_B		0x162650
+#define _CNL_PORT_TX_DW4_LN0_C		0x162C50
+#define _CNL_PORT_TX_DW4_LN0_D		0x162E50
+#define _CNL_PORT_TX_DW4_LN0_F		0x162850
+#define CNL_PORT_TX_DW4_GRP(port)       _MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW4_GRP_AE, \
+						    _CNL_PORT_TX_DW4_GRP_B, \
+						    _CNL_PORT_TX_DW4_GRP_C, \
+						    _CNL_PORT_TX_DW4_GRP_D, \
+						    _CNL_PORT_TX_DW4_GRP_AE, \
+						    _CNL_PORT_TX_DW4_GRP_F)
+#define CNL_PORT_TX_DW4_LN(port, ln)       _MMIO_PORT6_LN(port, ln,	\
+						    _CNL_PORT_TX_DW4_LN0_AE, \
+						    _CNL_PORT_TX_DW4_LN1_AE, \
+						    _CNL_PORT_TX_DW4_LN0_B, \
+						    _CNL_PORT_TX_DW4_LN0_C, \
+						    _CNL_PORT_TX_DW4_LN0_D, \
+						    _CNL_PORT_TX_DW4_LN0_AE, \
+						    _CNL_PORT_TX_DW4_LN0_F)
+#define   LOADGEN_SELECT		(1 << 31)
+#define   POST_CURSOR_1(x)		((x) << 12)
+#define   POST_CURSOR_1_MASK		(0x3F << 12)
+#define   POST_CURSOR_2(x)		((x) << 6)
+#define   POST_CURSOR_2_MASK		(0x3F << 6)
+#define   CURSOR_COEFF(x)		((x) << 0)
+#define   CURSOR_COEFF_MASK		(0x3F << 6)
+
+#define _CNL_PORT_TX_DW5_GRP_AE		0x162354
+#define _CNL_PORT_TX_DW5_GRP_B		0x1623D4
+#define _CNL_PORT_TX_DW5_GRP_C		0x162B54
+#define _CNL_PORT_TX_DW5_GRP_D		0x162BD4
+#define _CNL_PORT_TX_DW5_GRP_F		0x162A54
+#define _CNL_PORT_TX_DW5_LN0_AE		0x162454
+#define _CNL_PORT_TX_DW5_LN0_B		0x162654
+#define _CNL_PORT_TX_DW5_LN0_C		0x162C54
+#define _CNL_PORT_TX_DW5_LN0_D		0x162ED4
+#define _CNL_PORT_TX_DW5_LN0_F		0x162854
+#define CNL_PORT_TX_DW5_GRP(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW5_GRP_AE, \
+						    _CNL_PORT_TX_DW5_GRP_B, \
+						    _CNL_PORT_TX_DW5_GRP_C, \
+						    _CNL_PORT_TX_DW5_GRP_D, \
+						    _CNL_PORT_TX_DW5_GRP_AE, \
+						    _CNL_PORT_TX_DW5_GRP_F)
+#define CNL_PORT_TX_DW5_LN0(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW5_LN0_AE, \
+						    _CNL_PORT_TX_DW5_LN0_B, \
+						    _CNL_PORT_TX_DW5_LN0_C, \
+						    _CNL_PORT_TX_DW5_LN0_D, \
+						    _CNL_PORT_TX_DW5_LN0_AE, \
+						    _CNL_PORT_TX_DW5_LN0_F)
+#define   TX_TRAINING_EN		(1 << 31)
+#define   TAP3_DISABLE			(1 << 29)
+#define   SCALING_MODE_SEL(x)		((x) << 18)
+#define   SCALING_MODE_SEL_MASK		(0x7 << 18)
+#define   RTERM_SELECT(x)		((x) << 3)
+#define   RTERM_SELECT_MASK		(0x7 << 3)
+
+#define _CNL_PORT_TX_DW7_GRP_AE		0x16235C
+#define _CNL_PORT_TX_DW7_GRP_B		0x1623DC
+#define _CNL_PORT_TX_DW7_GRP_C		0x162B5C
+#define _CNL_PORT_TX_DW7_GRP_D		0x162BDC
+#define _CNL_PORT_TX_DW7_GRP_F		0x162A5C
+#define _CNL_PORT_TX_DW7_LN0_AE		0x16245C
+#define _CNL_PORT_TX_DW7_LN0_B		0x16265C
+#define _CNL_PORT_TX_DW7_LN0_C		0x162C5C
+#define _CNL_PORT_TX_DW7_LN0_D		0x162EDC
+#define _CNL_PORT_TX_DW7_LN0_F		0x16285C
+#define CNL_PORT_TX_DW7_GRP(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW7_GRP_AE, \
+						    _CNL_PORT_TX_DW7_GRP_B, \
+						    _CNL_PORT_TX_DW7_GRP_C, \
+						    _CNL_PORT_TX_DW7_GRP_D, \
+						    _CNL_PORT_TX_DW7_GRP_AE, \
+						    _CNL_PORT_TX_DW7_GRP_F)
+#define CNL_PORT_TX_DW7_LN0(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_TX_DW7_LN0_AE, \
+						    _CNL_PORT_TX_DW7_LN0_B, \
+						    _CNL_PORT_TX_DW7_LN0_C, \
+						    _CNL_PORT_TX_DW7_LN0_D, \
+						    _CNL_PORT_TX_DW7_LN0_AE, \
+						    _CNL_PORT_TX_DW7_LN0_F)
+#define   N_SCALAR(x)			((x) << 24)
+#define   N_SCALAR_MASK			(0x7F << 24)
+
 /* The spec defines this only for BXT PHY0, but lets assume that this
  * would exist for PHY1 too if it had a second channel.
  */
@@ -1682,6 +1870,23 @@ enum skl_disp_power_wells {
 #define BXT_PORT_CL2CM_DW6(phy)		_BXT_PHY((phy), _PORT_CL2CM_DW6_BC)
 #define   DW6_OLDO_DYN_PWR_DOWN_EN	(1 << 28)
 
+#define CNL_PORT_COMP_DW0		_MMIO(0x162100)
+#define   COMP_INIT			(1 << 31)
+#define CNL_PORT_COMP_DW1		_MMIO(0x162104)
+#define CNL_PORT_COMP_DW3		_MMIO(0x16210c)
+#define   PROCESS_INFO_DOT_0		(0 << 26)
+#define   PROCESS_INFO_DOT_1		(1 << 26)
+#define   PROCESS_INFO_DOT_4		(2 << 26)
+#define   PROCESS_INFO_MASK		(7 << 26)
+#define   PROCESS_INFO_SHIFT		26
+#define   VOLTAGE_INFO_0_85V		(0 << 24)
+#define   VOLTAGE_INFO_0_95V		(1 << 24)
+#define   VOLTAGE_INFO_1_05V		(2 << 24)
+#define   VOLTAGE_INFO_MASK		(3 << 24)
+#define   VOLTAGE_INFO_SHIFT		24
+#define CNL_PORT_COMP_DW9		_MMIO(0x162124)
+#define CNL_PORT_COMP_DW10		_MMIO(0x162128)
+
 /* BXT PHY Ref registers */
 #define _PORT_REF_DW3_A			0x16218C
 #define _PORT_REF_DW3_BC		0x6C18C
@@ -2317,6 +2522,9 @@ enum skl_disp_power_wells {
 #define   GEN8_RC_SEMA_IDLE_MSG_DISABLE	(1 << 12)
 #define   GEN8_FF_DOP_CLOCK_GATE_DISABLE	(1<<10)
 
+#define GEN6_RCS_PWR_FSM _MMIO(0x22ac)
+#define GEN9_RCS_FE_FSM2 _MMIO(0x22a4)
+
 /* Fuse readout registers for GT */
 #define CHV_FUSE_GT			_MMIO(VLV_DISPLAY_BASE + 0x2168)
 #define   CHV_FGT_DISABLE_SS0		(1 << 10)
@@ -2499,10 +2707,6 @@ enum skl_disp_power_wells {
 #define FBC_FENCE_OFF		_MMIO(0x3218) /* BSpec typo has 321Bh */
 #define FBC_TAG(i)		_MMIO(0x3300 + (i) * 4)
 
-#define FBC_STATUS2			_MMIO(0x43214)
-#define  IVB_FBC_COMPRESSION_MASK	0x7ff
-#define  BDW_FBC_COMPRESSION_MASK	0xfff
-
 #define FBC_LL_SIZE		(1536)
 
 #define FBC_LLC_READ_CTRL	_MMIO(0x9044)
@@ -2531,7 +2735,7 @@ enum skl_disp_power_wells {
 #define   DPFC_INVAL_SEG_SHIFT  (16)
 #define   DPFC_INVAL_SEG_MASK	(0x07ff0000)
 #define   DPFC_COMP_SEG_SHIFT	(0)
-#define   DPFC_COMP_SEG_MASK	(0x000003ff)
+#define   DPFC_COMP_SEG_MASK	(0x000007ff)
 #define DPFC_STATUS2		_MMIO(0x3214)
 #define DPFC_FENCE_YOFF		_MMIO(0x3218)
 #define DPFC_CHICKEN		_MMIO(0x3224)
@@ -2545,6 +2749,10 @@ enum skl_disp_power_wells {
 #define   DPFC_RESERVED		(0x1FFFFF00)
 #define ILK_DPFC_RECOMP_CTL	_MMIO(0x4320c)
 #define ILK_DPFC_STATUS		_MMIO(0x43210)
+#define  ILK_DPFC_COMP_SEG_MASK	0x7ff
+#define IVB_FBC_STATUS2		_MMIO(0x43214)
+#define  IVB_FBC_COMP_SEG_MASK	0x7ff
+#define  BDW_FBC_COMP_SEG_MASK	0xfff
 #define ILK_DPFC_FENCE_YOFF	_MMIO(0x43218)
 #define ILK_DPFC_CHICKEN	_MMIO(0x43224)
 #define   ILK_DPFC_DISABLE_DUMMY0 (1<<8)
@@ -2618,9 +2826,10 @@ enum skl_disp_power_wells {
 #define   GMBUS_PIN_DPB		5 /* SDVO, HDMIB */
 #define   GMBUS_PIN_DPD		6 /* HDMID */
 #define   GMBUS_PIN_RESERVED	7 /* 7 reserved */
-#define   GMBUS_PIN_1_BXT	1
+#define   GMBUS_PIN_1_BXT	1 /* BXT+ (atom) and CNP+ (big core) */
 #define   GMBUS_PIN_2_BXT	2
 #define   GMBUS_PIN_3_BXT	3
+#define   GMBUS_PIN_4_CNP	4
 #define   GMBUS_NUM_PINS	7 /* including 0 */
 #define GMBUS1			_MMIO(dev_priv->gpio_mmio_base + 0x5104) /* command/status */
 #define   GMBUS_SW_CLR_INT	(1<<31)
@@ -3366,16 +3575,6 @@ enum skl_disp_power_wells {
 #define GEN7_CXT_VFSTATE_SIZE(ctx_reg)	(((ctx_reg) >> 0) & 0x3f)
 #define GEN7_CXT_TOTAL_SIZE(ctx_reg)	(GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \
 					 GEN7_CXT_VFSTATE_SIZE(ctx_reg))
-/* Haswell does have the CXT_SIZE register however it does not appear to be
- * valid. Now, docs explain in dwords what is in the context object. The full
- * size is 70720 bytes, however, the power context and execlist context will
- * never be saved (power context is stored elsewhere, and execlists don't work
- * on HSW) - so the final size, including the extra state required for the
- * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
- */
-#define HSW_CXT_TOTAL_SIZE		(17 * PAGE_SIZE)
-/* Same as Haswell, but 72064 bytes now. */
-#define GEN8_CXT_TOTAL_SIZE		(18 * PAGE_SIZE)
 
 enum {
 	INTEL_ADVANCED_CONTEXT = 0,
@@ -5441,9 +5640,7 @@ enum {
 #define   CURSOR_MODE_128_ARGB_AX ((1 << 5) | CURSOR_MODE_128_32B_AX)
 #define   CURSOR_MODE_256_ARGB_AX ((1 << 5) | CURSOR_MODE_256_32B_AX)
 #define   CURSOR_MODE_64_ARGB_AX ((1 << 5) | CURSOR_MODE_64_32B_AX)
-#define   MCURSOR_PIPE_SELECT	(1 << 28)
-#define   MCURSOR_PIPE_A	0x00
-#define   MCURSOR_PIPE_B	(1 << 28)
+#define   MCURSOR_PIPE_SELECT(pipe)	((pipe) << 28)
 #define   MCURSOR_GAMMA_ENABLE  (1 << 26)
 #define   CURSOR_ROTATE_180	(1<<15)
 #define   CURSOR_TRICKLE_FEED_DISABLE	(1 << 14)
@@ -5453,7 +5650,9 @@ enum {
 #define   CURSOR_POS_SIGN       0x8000
 #define   CURSOR_X_SHIFT        0
 #define   CURSOR_Y_SHIFT        16
-#define CURSIZE			_MMIO(0x700a0)
+#define CURSIZE			_MMIO(0x700a0) /* 845/865 */
+#define _CUR_FBC_CTL_A		0x700a0 /* ivb+ */
+#define   CUR_FBC_CTL_EN	(1 << 31)
 #define _CURBCNTR		0x700c0
 #define _CURBBASE		0x700c4
 #define _CURBPOS		0x700c8
@@ -5469,6 +5668,7 @@ enum {
 #define CURCNTR(pipe) _CURSOR2(pipe, _CURACNTR)
 #define CURBASE(pipe) _CURSOR2(pipe, _CURABASE)
 #define CURPOS(pipe) _CURSOR2(pipe, _CURAPOS)
+#define CUR_FBC_CTL(pipe) _CURSOR2(pipe, _CUR_FBC_CTL_A)
 
 #define CURSOR_A_OFFSET 0x70080
 #define CURSOR_B_OFFSET 0x700c0
@@ -5501,8 +5701,7 @@ enum {
 #define   DISPPLANE_PIPE_CSC_ENABLE		(1<<24)
 #define   DISPPLANE_SEL_PIPE_SHIFT		24
 #define   DISPPLANE_SEL_PIPE_MASK		(3<<DISPPLANE_SEL_PIPE_SHIFT)
-#define   DISPPLANE_SEL_PIPE_A			0
-#define   DISPPLANE_SEL_PIPE_B			(1<<DISPPLANE_SEL_PIPE_SHIFT)
+#define   DISPPLANE_SEL_PIPE(pipe)		((pipe)<<DISPPLANE_SEL_PIPE_SHIFT)
 #define   DISPPLANE_SRC_KEY_ENABLE		(1<<22)
 #define   DISPPLANE_SRC_KEY_DISABLE		0
 #define   DISPPLANE_LINE_DOUBLE			(1<<20)
@@ -6508,6 +6707,9 @@ enum {
 #define  GLK_CL1_PWR_DOWN	(1 << 11)
 #define  GLK_CL2_PWR_DOWN	(1 << 12)
 
+#define CHICKEN_MISC_2		_MMIO(0x42084)
+#define  COMP_PWR_DOWN		(1 << 23)
+
 #define _CHICKEN_PIPESL_1_A	0x420b0
 #define _CHICKEN_PIPESL_1_B	0x420b4
 #define  HSW_FBCQ_DIS			(1 << 22)
@@ -6548,6 +6750,9 @@ enum {
 #define SKL_DFSM_PIPE_B_DISABLE		(1 << 21)
 #define SKL_DFSM_PIPE_C_DISABLE		(1 << 28)
 
+#define SKL_DSSM			_MMIO(0x51004)
+#define CNL_DSSM_CDCLK_PLL_REFCLK_24MHz	(1 << 31)
+
 #define GEN7_FF_SLICE_CS_CHICKEN1	_MMIO(0x20e0)
 #define   GEN9_FFSC_PERCTX_PREEMPT_CTRL	(1<<14)
 
@@ -6840,6 +7045,10 @@ enum {
 #define  FDL_TP2_TIMER_SHIFT    10
 #define  FDL_TP2_TIMER_MASK     (3<<10)
 #define  RAWCLK_FREQ_MASK       0x3ff
+#define  CNP_RAWCLK_DIV_MASK	(0x3ff << 16)
+#define  CNP_RAWCLK_DIV(div)	((div) << 16)
+#define  CNP_RAWCLK_FRAC_MASK	(0xf << 26)
+#define  CNP_RAWCLK_FRAC(frac)	((frac) << 26)
 
 #define PCH_DPLL_TMR_CFG        _MMIO(0xc6208)
 
@@ -7794,13 +8003,6 @@ enum {
 #define  SKL_FUSE_PG1_DIST_STATUS              (1<<26)
 #define  SKL_FUSE_PG2_DIST_STATUS              (1<<25)
 
-/* Decoupled MMIO register pair for kernel driver */
-#define GEN9_DECOUPLED_REG0_DW0			_MMIO(0xF00)
-#define GEN9_DECOUPLED_REG0_DW1			_MMIO(0xF04)
-#define GEN9_DECOUPLED_DW1_GO			(1<<31)
-#define GEN9_DECOUPLED_PD_SHIFT			28
-#define GEN9_DECOUPLED_OP_SHIFT			24
-
 /* Per-pipe DDI Function Control */
 #define _TRANS_DDI_FUNC_CTL_A		0x60400
 #define _TRANS_DDI_FUNC_CTL_B		0x61400
@@ -8109,6 +8311,61 @@ enum {
 #define DPLL_CFGCR1(id)	_MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR1, _DPLL2_CFGCR1)
 #define DPLL_CFGCR2(id)	_MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR2, _DPLL2_CFGCR2)
 
+/*
+ * CNL Clocks
+ */
+#define DPCLKA_CFGCR0				_MMIO(0x6C200)
+#define  DPCLKA_CFGCR0_DDI_CLK_OFF(port)	(1 << ((port)+10))
+#define  DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port)	(3 << ((port)*2))
+#define  DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)	((port)*2)
+#define  DPCLKA_CFGCR0_DDI_CLK_SEL(pll, port)	((pll) << ((port)*2))
+
+/* CNL PLL */
+#define DPLL0_ENABLE		0x46010
+#define DPLL1_ENABLE		0x46014
+#define  PLL_ENABLE		(1 << 31)
+#define  PLL_LOCK		(1 << 30)
+#define  PLL_POWER_ENABLE	(1 << 27)
+#define  PLL_POWER_STATE	(1 << 26)
+#define CNL_DPLL_ENABLE(pll)	_MMIO_PLL(pll, DPLL0_ENABLE, DPLL1_ENABLE)
+
+#define _CNL_DPLL0_CFGCR0		0x6C000
+#define _CNL_DPLL1_CFGCR0		0x6C080
+#define  DPLL_CFGCR0_HDMI_MODE		(1 << 30)
+#define  DPLL_CFGCR0_SSC_ENABLE		(1 << 29)
+#define  DPLL_CFGCR0_LINK_RATE_MASK	(0xf << 25)
+#define  DPLL_CFGCR0_LINK_RATE_2700	(0 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_1350	(1 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_810	(2 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_1620	(3 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_1080	(4 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_2160	(5 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_3240	(6 << 25)
+#define  DPLL_CFGCR0_LINK_RATE_4050	(7 << 25)
+#define  DPLL_CFGCR0_DCO_FRACTION_MASK	(0x7fff << 10)
+#define  DPLL_CFGCR0_DCO_FRACTION(x)	((x) << 10)
+#define  DPLL_CFGCR0_DCO_INTEGER_MASK	(0x3ff)
+#define CNL_DPLL_CFGCR0(pll)		_MMIO_PLL(pll, _CNL_DPLL0_CFGCR0, _CNL_DPLL1_CFGCR0)
+
+#define _CNL_DPLL0_CFGCR1		0x6C004
+#define _CNL_DPLL1_CFGCR1		0x6C084
+#define  DPLL_CFGCR1_QDIV_RATIO_MASK	(0xff << 10)
+#define  DPLL_CFGCR1_QDIV_RATIO(x)	((x) << 10)
+#define  DPLL_CFGCR1_QDIV_MODE(x)	((x) << 9)
+#define  DPLL_CFGCR1_KDIV_MASK		(7 << 6)
+#define  DPLL_CFGCR1_KDIV(x)		((x) << 6)
+#define  DPLL_CFGCR1_KDIV_1		(1 << 6)
+#define  DPLL_CFGCR1_KDIV_2		(2 << 6)
+#define  DPLL_CFGCR1_KDIV_4		(4 << 6)
+#define  DPLL_CFGCR1_PDIV_MASK		(0xf << 2)
+#define  DPLL_CFGCR1_PDIV(x)		((x) << 2)
+#define  DPLL_CFGCR1_PDIV_2		(1 << 2)
+#define  DPLL_CFGCR1_PDIV_3		(2 << 2)
+#define  DPLL_CFGCR1_PDIV_5		(4 << 2)
+#define  DPLL_CFGCR1_PDIV_7		(8 << 2)
+#define  DPLL_CFGCR1_CENTRAL_FREQ	(3 << 0)
+#define CNL_DPLL_CFGCR1(pll)		_MMIO_PLL(pll, _CNL_DPLL0_CFGCR1, _CNL_DPLL1_CFGCR1)
+
 /* BXT display engine PLL */
 #define BXT_DE_PLL_CTL			_MMIO(0x6d000)
 #define   BXT_DE_PLL_RATIO(x)		(x)	/* {60,65,100} * 19.2MHz */
@@ -8117,6 +8374,8 @@ enum {
 #define BXT_DE_PLL_ENABLE		_MMIO(0x46070)
 #define   BXT_DE_PLL_PLL_ENABLE		(1 << 31)
 #define   BXT_DE_PLL_LOCK		(1 << 30)
+#define   CNL_CDCLK_PLL_RATIO(x)	(x)
+#define   CNL_CDCLK_PLL_RATIO_MASK	0xff
 
 /* GEN9 DC */
 #define DC_STATE_EN			_MMIO(0x45504)
@@ -8150,6 +8409,7 @@ enum {
 /* SFUSE_STRAP */
 #define SFUSE_STRAP			_MMIO(0xc2014)
 #define  SFUSE_STRAP_FUSE_LOCK		(1<<13)
+#define  SFUSE_STRAP_RAW_FREQUENCY	(1<<8)
 #define  SFUSE_STRAP_DISPLAY_DISABLED	(1<<7)
 #define  SFUSE_STRAP_CRT_DISABLED	(1<<6)
 #define  SFUSE_STRAP_DDIB_DETECTED	(1<<2)
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
index 380de4360b8a..f29540f922af 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -12,6 +12,7 @@
 #include <linux/reservation.h>
 
 #include "i915_sw_fence.h"
+#include "i915_selftest.h"
 
 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
 
@@ -120,34 +121,6 @@ void i915_sw_fence_fini(struct i915_sw_fence *fence)
 }
 #endif
 
-static void i915_sw_fence_release(struct kref *kref)
-{
-	struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);
-
-	WARN_ON(atomic_read(&fence->pending) > 0);
-	debug_fence_destroy(fence);
-
-	if (fence->flags & I915_SW_FENCE_MASK) {
-		__i915_sw_fence_notify(fence, FENCE_FREE);
-	} else {
-		i915_sw_fence_fini(fence);
-		kfree(fence);
-	}
-}
-
-static void i915_sw_fence_put(struct i915_sw_fence *fence)
-{
-	debug_fence_assert(fence);
-	kref_put(&fence->kref, i915_sw_fence_release);
-}
-
-static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
-{
-	debug_fence_assert(fence);
-	kref_get(&fence->kref);
-	return fence;
-}
-
 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 					struct list_head *continuation)
 {
@@ -201,13 +174,15 @@ static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 
 	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 
-	if (fence->flags & I915_SW_FENCE_MASK &&
-	    __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
+	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 		return;
 
 	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 
 	__i915_sw_fence_wake_up_all(fence, continuation);
+
+	debug_fence_destroy(fence);
+	__i915_sw_fence_notify(fence, FENCE_FREE);
 }
 
 static void i915_sw_fence_complete(struct i915_sw_fence *fence)
@@ -231,33 +206,26 @@ void __i915_sw_fence_init(struct i915_sw_fence *fence,
 			  const char *name,
 			  struct lock_class_key *key)
 {
-	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
+	BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);
 
 	debug_fence_init(fence);
 
 	__init_waitqueue_head(&fence->wait, name, key);
-	kref_init(&fence->kref);
 	atomic_set(&fence->pending, 1);
 	fence->flags = (unsigned long)fn;
 }
 
-static void __i915_sw_fence_commit(struct i915_sw_fence *fence)
-{
-	i915_sw_fence_complete(fence);
-	i915_sw_fence_put(fence);
-}
-
 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 {
 	debug_fence_activate(fence);
-	__i915_sw_fence_commit(fence);
+	i915_sw_fence_complete(fence);
 }
 
 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 {
 	list_del(&wq->entry);
 	__i915_sw_fence_complete(wq->private, key);
-	i915_sw_fence_put(wq->private);
+
 	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 		kfree(wq);
 	return 0;
@@ -306,7 +274,7 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 	unsigned long flags;
 	bool err;
 
-	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
+	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
 		return false;
 
 	spin_lock_irqsave(&i915_sw_fence_lock, flags);
@@ -352,7 +320,7 @@ static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 	INIT_LIST_HEAD(&wq->entry);
 	wq->flags = pending;
 	wq->func = i915_sw_fence_wake;
-	wq->private = i915_sw_fence_get(fence);
+	wq->private = fence;
 
 	i915_sw_fence_await(fence);
 
@@ -401,7 +369,7 @@ static void timer_i915_sw_fence_wake(unsigned long data)
 	dma_fence_put(cb->dma);
 	cb->dma = NULL;
 
-	__i915_sw_fence_commit(cb->fence);
+	i915_sw_fence_complete(cb->fence);
 	cb->timer.function = NULL;
 }
 
@@ -412,7 +380,7 @@ static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 
 	del_timer_sync(&cb->timer);
 	if (cb->timer.function)
-		__i915_sw_fence_commit(cb->fence);
+		i915_sw_fence_complete(cb->fence);
 	dma_fence_put(cb->dma);
 
 	kfree(cb);
@@ -439,7 +407,7 @@ int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 		return dma_fence_wait(dma, false);
 	}
 
-	cb->fence = i915_sw_fence_get(fence);
+	cb->fence = fence;
 	i915_sw_fence_await(fence);
 
 	cb->dma = NULL;
@@ -522,3 +490,7 @@ int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 
 	return ret;
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/i915_sw_fence.c"
+#endif
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h
index fd3c3bf6c8b7..fe2ef4dadfc6 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.h
+++ b/drivers/gpu/drm/i915/i915_sw_fence.h
@@ -23,7 +23,6 @@ struct reservation_object;
 struct i915_sw_fence {
 	wait_queue_head_t wait;
 	unsigned long flags;
-	struct kref kref;
 	atomic_t pending;
 };
 
diff --git a/drivers/gpu/drm/i915/i915_syncmap.c b/drivers/gpu/drm/i915/i915_syncmap.c
new file mode 100644
index 000000000000..0087acf731a8
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_syncmap.c
@@ -0,0 +1,412 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "i915_syncmap.h"
+
+#include "i915_gem.h" /* GEM_BUG_ON() */
+#include "i915_selftest.h"
+
+#define SHIFT ilog2(KSYNCMAP)
+#define MASK (KSYNCMAP - 1)
+
+/*
+ * struct i915_syncmap is a layer of a radixtree that maps a u64 fence
+ * context id to the last u32 fence seqno waited upon from that context.
+ * Unlike lib/radixtree it uses a parent pointer that allows traversal back to
+ * the root. This allows us to access the whole tree via a single pointer
+ * to the most recently used layer. We expect fence contexts to be dense
+ * and most reuse to be on the same i915_gem_context but on neighbouring
+ * engines (i.e. on adjacent contexts) and reuse the same leaf, a very
+ * effective lookup cache. If the new lookup is not on the same leaf, we
+ * expect it to be on the neighbouring branch.
+ *
+ * A leaf holds an array of u32 seqno, and has height 0. The bitmap field
+ * allows us to store whether a particular seqno is valid (i.e. allows us
+ * to distinguish unset from 0).
+ *
+ * A branch holds an array of layer pointers, and has height > 0, and always
+ * has at least 2 layers (either branches or leaves) below it.
+ *
+ * For example,
+ *	for x in
+ *	  0 1 2 0x10 0x11 0x200 0x201
+ *	  0x500000 0x500001 0x503000 0x503001
+ *	  0xE<<60:
+ *		i915_syncmap_set(&sync, x, lower_32_bits(x));
+ * will build a tree like:
+ *	0xXXXXXXXXXXXXXXXX
+ *	0-> 0x0000000000XXXXXX
+ *	|   0-> 0x0000000000000XXX
+ *	|   |   0-> 0x00000000000000XX
+ *	|   |   |   0-> 0x000000000000000X 0:0, 1:1, 2:2
+ *	|   |   |   1-> 0x000000000000001X 0:10, 1:11
+ *	|   |   2-> 0x000000000000020X 0:200, 1:201
+ *	|   5-> 0x000000000050XXXX
+ *	|       0-> 0x000000000050000X 0:500000, 1:500001
+ *	|       3-> 0x000000000050300X 0:503000, 1:503001
+ *	e-> 0xe00000000000000X e:e
+ */
+
+struct i915_syncmap {
+	u64 prefix;
+	unsigned int height;
+	unsigned int bitmap;
+	struct i915_syncmap *parent;
+	/*
+	 * Following this header is an array of either seqno or child pointers:
+	 * union {
+	 *	u32 seqno[KSYNCMAP];
+	 *	struct i915_syncmap *child[KSYNCMAP];
+	 * };
+	 */
+};
+
+/**
+ * i915_syncmap_init -- initialise the #i915_syncmap
+ * @root - pointer to the #i915_syncmap
+ */
+void i915_syncmap_init(struct i915_syncmap **root)
+{
+	BUILD_BUG_ON_NOT_POWER_OF_2(KSYNCMAP);
+	BUILD_BUG_ON_NOT_POWER_OF_2(SHIFT);
+	BUILD_BUG_ON(KSYNCMAP > BITS_PER_BYTE * sizeof((*root)->bitmap));
+	*root = NULL;
+}
+
+static inline u32 *__sync_seqno(struct i915_syncmap *p)
+{
+	GEM_BUG_ON(p->height);
+	return (u32 *)(p + 1);
+}
+
+static inline struct i915_syncmap **__sync_child(struct i915_syncmap *p)
+{
+	GEM_BUG_ON(!p->height);
+	return (struct i915_syncmap **)(p + 1);
+}
+
+static inline unsigned int
+__sync_branch_idx(const struct i915_syncmap *p, u64 id)
+{
+	return (id >> p->height) & MASK;
+}
+
+static inline unsigned int
+__sync_leaf_idx(const struct i915_syncmap *p, u64 id)
+{
+	GEM_BUG_ON(p->height);
+	return id & MASK;
+}
+
+static inline u64 __sync_branch_prefix(const struct i915_syncmap *p, u64 id)
+{
+	return id >> p->height >> SHIFT;
+}
+
+static inline u64 __sync_leaf_prefix(const struct i915_syncmap *p, u64 id)
+{
+	GEM_BUG_ON(p->height);
+	return id >> SHIFT;
+}
+
+static inline bool seqno_later(u32 a, u32 b)
+{
+	return (s32)(a - b) >= 0;
+}
+
+/**
+ * i915_syncmap_is_later -- compare against the last know sync point
+ * @root - pointer to the #i915_syncmap
+ * @id - the context id (other timeline) we are synchronising to
+ * @seqno - the sequence number along the other timeline
+ *
+ * If we have already synchronised this @root timeline with another (@id) then
+ * we can omit any repeated or earlier synchronisation requests. If the two
+ * timelines are already coupled, we can also omit the dependency between the
+ * two as that is already known via the timeline.
+ *
+ * Returns true if the two timelines are already synchronised wrt to @seqno,
+ * false if not and the synchronisation must be emitted.
+ */
+bool i915_syncmap_is_later(struct i915_syncmap **root, u64 id, u32 seqno)
+{
+	struct i915_syncmap *p;
+	unsigned int idx;
+
+	p = *root;
+	if (!p)
+		return false;
+
+	if (likely(__sync_leaf_prefix(p, id) == p->prefix))
+		goto found;
+
+	/* First climb the tree back to a parent branch */
+	do {
+		p = p->parent;
+		if (!p)
+			return false;
+
+		if (__sync_branch_prefix(p, id) == p->prefix)
+			break;
+	} while (1);
+
+	/* And then descend again until we find our leaf */
+	do {
+		if (!p->height)
+			break;
+
+		p = __sync_child(p)[__sync_branch_idx(p, id)];
+		if (!p)
+			return false;
+
+		if (__sync_branch_prefix(p, id) != p->prefix)
+			return false;
+	} while (1);
+
+	*root = p;
+found:
+	idx = __sync_leaf_idx(p, id);
+	if (!(p->bitmap & BIT(idx)))
+		return false;
+
+	return seqno_later(__sync_seqno(p)[idx], seqno);
+}
+
+static struct i915_syncmap *
+__sync_alloc_leaf(struct i915_syncmap *parent, u64 id)
+{
+	struct i915_syncmap *p;
+
+	p = kmalloc(sizeof(*p) + KSYNCMAP * sizeof(u32), GFP_KERNEL);
+	if (unlikely(!p))
+		return NULL;
+
+	p->parent = parent;
+	p->height = 0;
+	p->bitmap = 0;
+	p->prefix = __sync_leaf_prefix(p, id);
+	return p;
+}
+
+static inline void __sync_set_seqno(struct i915_syncmap *p, u64 id, u32 seqno)
+{
+	unsigned int idx = __sync_leaf_idx(p, id);
+
+	p->bitmap |= BIT(idx);
+	__sync_seqno(p)[idx] = seqno;
+}
+
+static inline void __sync_set_child(struct i915_syncmap *p,
+				    unsigned int idx,
+				    struct i915_syncmap *child)
+{
+	p->bitmap |= BIT(idx);
+	__sync_child(p)[idx] = child;
+}
+
+static noinline int __sync_set(struct i915_syncmap **root, u64 id, u32 seqno)
+{
+	struct i915_syncmap *p = *root;
+	unsigned int idx;
+
+	if (!p) {
+		p = __sync_alloc_leaf(NULL, id);
+		if (unlikely(!p))
+			return -ENOMEM;
+
+		goto found;
+	}
+
+	/* Caller handled the likely cached case */
+	GEM_BUG_ON(__sync_leaf_prefix(p, id) == p->prefix);
+
+	/* Climb back up the tree until we find a common prefix */
+	do {
+		if (!p->parent)
+			break;
+
+		p = p->parent;
+
+		if (__sync_branch_prefix(p, id) == p->prefix)
+			break;
+	} while (1);
+
+	/*
+	 * No shortcut, we have to descend the tree to find the right layer
+	 * containing this fence.
+	 *
+	 * Each layer in the tree holds 16 (KSYNCMAP) pointers, either fences
+	 * or lower layers. Leaf nodes (height = 0) contain the fences, all
+	 * other nodes (height > 0) are internal layers that point to a lower
+	 * node. Each internal layer has at least 2 descendents.
+	 *
+	 * Starting at the top, we check whether the current prefix matches. If
+	 * it doesn't, we have gone past our target and need to insert a join
+	 * into the tree, and a new leaf node for the target as a descendent
+	 * of the join, as well as the original layer.
+	 *
+	 * The matching prefix means we are still following the right branch
+	 * of the tree. If it has height 0, we have found our leaf and just
+	 * need to replace the fence slot with ourselves. If the height is
+	 * not zero, our slot contains the next layer in the tree (unless
+	 * it is empty, in which case we can add ourselves as a new leaf).
+	 * As descend the tree the prefix grows (and height decreases).
+	 */
+	do {
+		struct i915_syncmap *next;
+
+		if (__sync_branch_prefix(p, id) != p->prefix) {
+			unsigned int above;
+
+			/* Insert a join above the current layer */
+			next = kzalloc(sizeof(*next) + KSYNCMAP * sizeof(next),
+				       GFP_KERNEL);
+			if (unlikely(!next))
+				return -ENOMEM;
+
+			/* Compute the height at which these two diverge */
+			above = fls64(__sync_branch_prefix(p, id) ^ p->prefix);
+			above = round_up(above, SHIFT);
+			next->height = above + p->height;
+			next->prefix = __sync_branch_prefix(next, id);
+
+			/* Insert the join into the parent */
+			if (p->parent) {
+				idx = __sync_branch_idx(p->parent, id);
+				__sync_child(p->parent)[idx] = next;
+				GEM_BUG_ON(!(p->parent->bitmap & BIT(idx)));
+			}
+			next->parent = p->parent;
+
+			/* Compute the idx of the other branch, not our id! */
+			idx = p->prefix >> (above - SHIFT) & MASK;
+			__sync_set_child(next, idx, p);
+			p->parent = next;
+
+			/* Ascend to the join */
+			p = next;
+		} else {
+			if (!p->height)
+				break;
+		}
+
+		/* Descend into the next layer */
+		GEM_BUG_ON(!p->height);
+		idx = __sync_branch_idx(p, id);
+		next = __sync_child(p)[idx];
+		if (!next) {
+			next = __sync_alloc_leaf(p, id);
+			if (unlikely(!next))
+				return -ENOMEM;
+
+			__sync_set_child(p, idx, next);
+			p = next;
+			break;
+		}
+
+		p = next;
+	} while (1);
+
+found:
+	GEM_BUG_ON(p->prefix != __sync_leaf_prefix(p, id));
+	__sync_set_seqno(p, id, seqno);
+	*root = p;
+	return 0;
+}
+
+/**
+ * i915_syncmap_set -- mark the most recent syncpoint between contexts
+ * @root - pointer to the #i915_syncmap
+ * @id - the context id (other timeline) we have synchronised to
+ * @seqno - the sequence number along the other timeline
+ *
+ * When we synchronise this @root timeline with another (@id), we also know
+ * that we have synchronized with all previous seqno along that timeline. If
+ * we then have a request to synchronise with the same seqno or older, we can
+ * omit it, see i915_syncmap_is_later()
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int i915_syncmap_set(struct i915_syncmap **root, u64 id, u32 seqno)
+{
+	struct i915_syncmap *p = *root;
+
+	/*
+	 * We expect to be called in sequence following is_later(id), which
+	 * should have preloaded the root for us.
+	 */
+	if (likely(p && __sync_leaf_prefix(p, id) == p->prefix)) {
+		__sync_set_seqno(p, id, seqno);
+		return 0;
+	}
+
+	return __sync_set(root, id, seqno);
+}
+
+static void __sync_free(struct i915_syncmap *p)
+{
+	if (p->height) {
+		unsigned int i;
+
+		while ((i = ffs(p->bitmap))) {
+			p->bitmap &= ~0u << i;
+			__sync_free(__sync_child(p)[i - 1]);
+		}
+	}
+
+	kfree(p);
+}
+
+/**
+ * i915_syncmap_free -- free all memory associated with the syncmap
+ * @root - pointer to the #i915_syncmap
+ *
+ * Either when the timeline is to be freed and we no longer need the sync
+ * point tracking, or when the fences are all known to be signaled and the
+ * sync point tracking is redundant, we can free the #i915_syncmap to recover
+ * its allocations.
+ *
+ * Will reinitialise the @root pointer so that the #i915_syncmap is ready for
+ * reuse.
+ */
+void i915_syncmap_free(struct i915_syncmap **root)
+{
+	struct i915_syncmap *p;
+
+	p = *root;
+	if (!p)
+		return;
+
+	while (p->parent)
+		p = p->parent;
+
+	__sync_free(p);
+	*root = NULL;
+}
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/i915_syncmap.c"
+#endif
diff --git a/drivers/gpu/drm/i915/i915_syncmap.h b/drivers/gpu/drm/i915/i915_syncmap.h
new file mode 100644
index 000000000000..0653f70bee82
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_syncmap.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_SYNCMAP_H__
+#define __I915_SYNCMAP_H__
+
+#include <linux/types.h>
+
+struct i915_syncmap;
+#define KSYNCMAP 16 /* radix of the tree, how many slots in each layer */
+
+void i915_syncmap_init(struct i915_syncmap **root);
+int i915_syncmap_set(struct i915_syncmap **root, u64 id, u32 seqno);
+bool i915_syncmap_is_later(struct i915_syncmap **root, u64 id, u32 seqno);
+void i915_syncmap_free(struct i915_syncmap **root);
+
+#endif /* __I915_SYNCMAP_H__ */
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index f3fdfda5e558..1eef3fae4db3 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -181,13 +181,10 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
 	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 	struct drm_device *dev = &dev_priv->drm;
 	struct i915_gem_context *ctx;
-	u32 *temp = NULL; /* Just here to make handling failures easy */
 	int slice = (int)(uintptr_t)attr->private;
+	u32 **remap_info;
 	int ret;
 
-	if (!HAS_HW_CONTEXTS(dev_priv))
-		return -ENXIO;
-
 	ret = l3_access_valid(dev_priv, offset);
 	if (ret)
 		return ret;
@@ -196,11 +193,12 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
 	if (ret)
 		return ret;
 
-	if (!dev_priv->l3_parity.remap_info[slice]) {
-		temp = kzalloc(GEN7_L3LOG_SIZE, GFP_KERNEL);
-		if (!temp) {
-			mutex_unlock(&dev->struct_mutex);
-			return -ENOMEM;
+	remap_info = &dev_priv->l3_parity.remap_info[slice];
+	if (!*remap_info) {
+		*remap_info = kzalloc(GEN7_L3LOG_SIZE, GFP_KERNEL);
+		if (!*remap_info) {
+			ret = -ENOMEM;
+			goto out;
 		}
 	}
 
@@ -208,18 +206,18 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
 	 * aren't propagated. Since I cannot find a stable way to reset the GPU
 	 * at this point it is left as a TODO.
 	*/
-	if (temp)
-		dev_priv->l3_parity.remap_info[slice] = temp;
-
-	memcpy(dev_priv->l3_parity.remap_info[slice] + (offset/4), buf, count);
+	memcpy(*remap_info + (offset/4), buf, count);
 
 	/* NB: We defer the remapping until we switch to the context */
 	list_for_each_entry(ctx, &dev_priv->context_list, link)
 		ctx->remap_slice |= (1<<slice);
 
+	ret = count;
+
+out:
 	mutex_unlock(&dev->struct_mutex);
 
-	return count;
+	return ret;
 }
 
 static struct bin_attribute dpf_attrs = {
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 66404c5aee82..b24a83d43559 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -89,6 +89,55 @@ TRACE_EVENT(intel_memory_cxsr,
 		      __entry->frame[PIPE_C], __entry->scanline[PIPE_C])
 );
 
+TRACE_EVENT(g4x_wm,
+	    TP_PROTO(struct intel_crtc *crtc, const struct g4x_wm_values *wm),
+	    TP_ARGS(crtc, wm),
+
+	    TP_STRUCT__entry(
+			     __field(enum pipe, pipe)
+			     __field(u32, frame)
+			     __field(u32, scanline)
+			     __field(u16, primary)
+			     __field(u16, sprite)
+			     __field(u16, cursor)
+			     __field(u16, sr_plane)
+			     __field(u16, sr_cursor)
+			     __field(u16, sr_fbc)
+			     __field(u16, hpll_plane)
+			     __field(u16, hpll_cursor)
+			     __field(u16, hpll_fbc)
+			     __field(bool, cxsr)
+			     __field(bool, hpll)
+			     __field(bool, fbc)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->pipe = crtc->pipe;
+			   __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev,
+										       crtc->pipe);
+			   __entry->scanline = intel_get_crtc_scanline(crtc);
+			   __entry->primary = wm->pipe[crtc->pipe].plane[PLANE_PRIMARY];
+			   __entry->sprite = wm->pipe[crtc->pipe].plane[PLANE_SPRITE0];
+			   __entry->cursor = wm->pipe[crtc->pipe].plane[PLANE_CURSOR];
+			   __entry->sr_plane = wm->sr.plane;
+			   __entry->sr_cursor = wm->sr.cursor;
+			   __entry->sr_fbc = wm->sr.fbc;
+			   __entry->hpll_plane = wm->hpll.plane;
+			   __entry->hpll_cursor = wm->hpll.cursor;
+			   __entry->hpll_fbc = wm->hpll.fbc;
+			   __entry->cxsr = wm->cxsr;
+			   __entry->hpll = wm->hpll_en;
+			   __entry->fbc = wm->fbc_en;
+			   ),
+
+	    TP_printk("pipe %c, frame=%u, scanline=%u, wm %d/%d/%d, sr %s/%d/%d/%d, hpll %s/%d/%d/%d, fbc %s",
+		      pipe_name(__entry->pipe), __entry->frame, __entry->scanline,
+		      __entry->primary, __entry->sprite, __entry->cursor,
+		      yesno(__entry->cxsr), __entry->sr_plane, __entry->sr_cursor, __entry->sr_fbc,
+		      yesno(__entry->hpll), __entry->hpll_plane, __entry->hpll_cursor, __entry->hpll_fbc,
+		      yesno(__entry->fbc))
+);
+
 TRACE_EVENT(vlv_wm,
 	    TP_PROTO(struct intel_crtc *crtc, const struct vlv_wm_values *wm),
 	    TP_ARGS(crtc, wm),
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index c5455d36b617..12fc250b47b9 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -70,20 +70,27 @@
 #define overflows_type(x, T) \
 	(sizeof(x) > sizeof(T) && (x) >> (sizeof(T) * BITS_PER_BYTE))
 
-#define ptr_mask_bits(ptr) ({						\
+#define ptr_mask_bits(ptr, n) ({					\
 	unsigned long __v = (unsigned long)(ptr);			\
-	(typeof(ptr))(__v & PAGE_MASK);					\
+	(typeof(ptr))(__v & -BIT(n));					\
 })
 
-#define ptr_unpack_bits(ptr, bits) ({					\
+#define ptr_unmask_bits(ptr, n) ((unsigned long)(ptr) & (BIT(n) - 1))
+
+#define ptr_unpack_bits(ptr, bits, n) ({				\
 	unsigned long __v = (unsigned long)(ptr);			\
-	(bits) = __v & ~PAGE_MASK;					\
-	(typeof(ptr))(__v & PAGE_MASK);					\
+	*(bits) = __v & (BIT(n) - 1);					\
+	(typeof(ptr))(__v & -BIT(n));					\
 })
 
-#define ptr_pack_bits(ptr, bits)					\
+#define ptr_pack_bits(ptr, bits, n)					\
 	((typeof(ptr))((unsigned long)(ptr) | (bits)))
 
+#define page_mask_bits(ptr) ptr_mask_bits(ptr, PAGE_SHIFT)
+#define page_unmask_bits(ptr) ptr_unmask_bits(ptr, PAGE_SHIFT)
+#define page_pack_bits(ptr, bits) ptr_pack_bits(ptr, bits, PAGE_SHIFT)
+#define page_unpack_bits(ptr, bits) ptr_unpack_bits(ptr, bits, PAGE_SHIFT)
+
 #define ptr_offset(ptr, member) offsetof(typeof(*(ptr)), member)
 
 #define fetch_and_zero(ptr) ({						\
@@ -92,4 +99,24 @@
 	__T;								\
 })
 
+#define u64_to_ptr(T, x) ({						\
+	typecheck(u64, x);						\
+	(T *)(uintptr_t)(x);						\
+})
+
+#define __mask_next_bit(mask) ({					\
+	int __idx = ffs(mask) - 1;					\
+	mask &= ~BIT(__idx);						\
+	__idx;								\
+})
+
+#include <linux/list.h>
+
+static inline void __list_del_many(struct list_head *head,
+				   struct list_head *first)
+{
+	first->prev = head;
+	WRITE_ONCE(head->next, first);
+}
+
 #endif /* !__I915_UTILS_H */
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index 2e739018fb4c..cf7a958e4d3c 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -90,6 +90,18 @@ struct _balloon_info_ {
 
 static struct _balloon_info_ bl_info;
 
+static void vgt_deballoon_space(struct i915_ggtt *ggtt,
+				struct drm_mm_node *node)
+{
+	DRM_DEBUG_DRIVER("deballoon space: range [0x%llx - 0x%llx] %llu KiB.\n",
+			 node->start,
+			 node->start + node->size,
+			 node->size / 1024);
+
+	ggtt->base.reserved -= node->size;
+	drm_mm_remove_node(node);
+}
+
 /**
  * intel_vgt_deballoon - deballoon reserved graphics address trunks
  * @dev_priv: i915 device private data
@@ -106,12 +118,8 @@ void intel_vgt_deballoon(struct drm_i915_private *dev_priv)
 
 	DRM_DEBUG("VGT deballoon.\n");
 
-	for (i = 0; i < 4; i++) {
-		if (bl_info.space[i].allocated)
-			drm_mm_remove_node(&bl_info.space[i]);
-	}
-
-	memset(&bl_info, 0, sizeof(bl_info));
+	for (i = 0; i < 4; i++)
+		vgt_deballoon_space(&dev_priv->ggtt, &bl_info.space[i]);
 }
 
 static int vgt_balloon_space(struct i915_ggtt *ggtt,
@@ -119,15 +127,20 @@ static int vgt_balloon_space(struct i915_ggtt *ggtt,
 			     unsigned long start, unsigned long end)
 {
 	unsigned long size = end - start;
+	int ret;
 
 	if (start >= end)
 		return -EINVAL;
 
 	DRM_INFO("balloon space: range [ 0x%lx - 0x%lx ] %lu KiB.\n",
 		 start, end, size / 1024);
-	return i915_gem_gtt_reserve(&ggtt->base, node,
-				    size, start, I915_COLOR_UNEVICTABLE,
-				    0);
+	ret = i915_gem_gtt_reserve(&ggtt->base, node,
+				   size, start, I915_COLOR_UNEVICTABLE,
+				   0);
+	if (!ret)
+		ggtt->base.reserved += size;
+
+	return ret;
 }
 
 /**
@@ -220,7 +233,7 @@ int intel_vgt_balloon(struct drm_i915_private *dev_priv)
 		ret = vgt_balloon_space(ggtt, &bl_info.space[3],
 					unmappable_end, ggtt_end);
 		if (ret)
-			goto err;
+			goto err_upon_mappable;
 	}
 
 	/* Mappable graphic memory ballooning */
@@ -229,7 +242,7 @@ int intel_vgt_balloon(struct drm_i915_private *dev_priv)
 					0, mappable_base);
 
 		if (ret)
-			goto err;
+			goto err_upon_unmappable;
 	}
 
 	if (mappable_end < ggtt->mappable_end) {
@@ -237,14 +250,19 @@ int intel_vgt_balloon(struct drm_i915_private *dev_priv)
 					mappable_end, ggtt->mappable_end);
 
 		if (ret)
-			goto err;
+			goto err_below_mappable;
 	}
 
 	DRM_INFO("VGT balloon successfully\n");
 	return 0;
 
+err_below_mappable:
+	vgt_deballoon_space(ggtt, &bl_info.space[0]);
+err_upon_unmappable:
+	vgt_deballoon_space(ggtt, &bl_info.space[3]);
+err_upon_mappable:
+	vgt_deballoon_space(ggtt, &bl_info.space[2]);
 err:
 	DRM_ERROR("VGT balloon fail\n");
-	intel_vgt_deballoon(dev_priv);
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index f066e2d785f5..1cfe137cdc32 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -85,12 +85,12 @@ vma_create(struct drm_i915_gem_object *obj,
 	if (vma == NULL)
 		return ERR_PTR(-ENOMEM);
 
-	INIT_LIST_HEAD(&vma->exec_list);
 	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
 		init_request_active(&vma->last_read[i], i915_vma_retire);
 	init_request_active(&vma->last_fence, NULL);
 	vma->vm = vm;
 	vma->obj = obj;
+	vma->resv = obj->resv;
 	vma->size = obj->base.size;
 	vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
 
@@ -464,7 +464,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 			  size, obj->base.size,
 			  flags & PIN_MAPPABLE ? "mappable" : "total",
 			  end);
-		return -E2BIG;
+		return -ENOSPC;
 	}
 
 	ret = i915_gem_object_pin_pages(obj);
@@ -577,7 +577,7 @@ err_unpin:
 	return ret;
 }
 
-void i915_vma_destroy(struct i915_vma *vma)
+static void i915_vma_destroy(struct i915_vma *vma)
 {
 	GEM_BUG_ON(vma->node.allocated);
 	GEM_BUG_ON(i915_vma_is_active(vma));
@@ -591,11 +591,33 @@ void i915_vma_destroy(struct i915_vma *vma)
 	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
 }
 
+void i915_vma_unlink_ctx(struct i915_vma *vma)
+{
+	struct i915_gem_context *ctx = vma->ctx;
+
+	if (ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
+		cancel_work_sync(&ctx->vma_lut.resize);
+		ctx->vma_lut.ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+	}
+
+	__hlist_del(&vma->ctx_node);
+	ctx->vma_lut.ht_count--;
+
+	if (i915_vma_is_ggtt(vma))
+		vma->obj->vma_hashed = NULL;
+	vma->ctx = NULL;
+
+	i915_vma_put(vma);
+}
+
 void i915_vma_close(struct i915_vma *vma)
 {
 	GEM_BUG_ON(i915_vma_is_closed(vma));
 	vma->flags |= I915_VMA_CLOSED;
 
+	if (vma->ctx)
+		i915_vma_unlink_ctx(vma);
+
 	list_del(&vma->obj_link);
 	rb_erase(&vma->obj_node, &vma->obj->vma_tree);
 
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 2e03f81dddbe..4a673fc1a432 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -50,6 +50,7 @@ struct i915_vma {
 	struct drm_i915_gem_object *obj;
 	struct i915_address_space *vm;
 	struct drm_i915_fence_reg *fence;
+	struct reservation_object *resv; /** Alias of obj->resv */
 	struct sg_table *pages;
 	void __iomem *iomap;
 	u64 size;
@@ -99,16 +100,25 @@ struct i915_vma {
 
 	struct list_head obj_link; /* Link in the object's VMA list */
 	struct rb_node obj_node;
+	struct hlist_node obj_hash;
 
-	/** This vma's place in the batchbuffer or on the eviction list */
-	struct list_head exec_list;
+	/** This vma's place in the execbuf reservation list */
+	struct list_head exec_link;
+	struct list_head reloc_link;
+
+	/** This vma's place in the eviction list */
+	struct list_head evict_link;
 
 	/**
 	 * Used for performing relocations during execbuffer insertion.
 	 */
-	struct hlist_node exec_node;
-	unsigned long exec_handle;
 	struct drm_i915_gem_exec_object2 *exec_entry;
+	struct hlist_node exec_node;
+	u32 exec_handle;
+
+	struct i915_gem_context *ctx;
+	struct hlist_node ctx_node;
+	u32 ctx_handle;
 };
 
 struct i915_vma *
@@ -232,8 +242,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma,
 			u64 size, u64 alignment, u64 flags);
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
+void i915_vma_unlink_ctx(struct i915_vma *vma);
 void i915_vma_close(struct i915_vma *vma);
-void i915_vma_destroy(struct i915_vma *vma);
 
 int __i915_vma_do_pin(struct i915_vma *vma,
 		      u64 size, u64 alignment, u64 flags);
diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c
index 50fb1f76cc5f..36d4e635e4ce 100644
--- a/drivers/gpu/drm/i915/intel_atomic.c
+++ b/drivers/gpu/drm/i915/intel_atomic.c
@@ -36,44 +36,121 @@
 #include "intel_drv.h"
 
 /**
- * intel_connector_atomic_get_property - fetch connector property value
- * @connector: connector to fetch property for
- * @state: state containing the property value
- * @property: property to look up
- * @val: pointer to write property value into
+ * intel_digital_connector_atomic_get_property - hook for connector->atomic_get_property.
+ * @connector: Connector to get the property for.
+ * @state: Connector state to retrieve the property from.
+ * @property: Property to retrieve.
+ * @val: Return value for the property.
  *
- * The DRM core does not store shadow copies of properties for
- * atomic-capable drivers.  This entrypoint is used to fetch
- * the current value of a driver-specific connector property.
+ * Returns the atomic property value for a digital connector.
  */
-int
-intel_connector_atomic_get_property(struct drm_connector *connector,
-				    const struct drm_connector_state *state,
-				    struct drm_property *property,
-				    uint64_t *val)
+int intel_digital_connector_atomic_get_property(struct drm_connector *connector,
+						const struct drm_connector_state *state,
+						struct drm_property *property,
+						uint64_t *val)
 {
-	int i;
+	struct drm_device *dev = connector->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_digital_connector_state *intel_conn_state =
+		to_intel_digital_connector_state(state);
+
+	if (property == dev_priv->force_audio_property)
+		*val = intel_conn_state->force_audio;
+	else if (property == dev_priv->broadcast_rgb_property)
+		*val = intel_conn_state->broadcast_rgb;
+	else {
+		DRM_DEBUG_ATOMIC("Unknown property %s\n", property->name);
+		return -EINVAL;
+	}
 
-	/*
-	 * TODO: We only have atomic modeset for planes at the moment, so the
-	 * crtc/connector code isn't quite ready yet.  Until it's ready,
-	 * continue to look up all property values in the DRM's shadow copy
-	 * in obj->properties->values[].
-	 *
-	 * When the crtc/connector state work matures, this function should
-	 * be updated to read the values out of the state structure instead.
-	 */
-	for (i = 0; i < connector->base.properties->count; i++) {
-		if (connector->base.properties->properties[i] == property) {
-			*val = connector->base.properties->values[i];
-			return 0;
-		}
+	return 0;
+}
+
+/**
+ * intel_digital_connector_atomic_set_property - hook for connector->atomic_set_property.
+ * @connector: Connector to set the property for.
+ * @state: Connector state to set the property on.
+ * @property: Property to set.
+ * @val: New value for the property.
+ *
+ * Sets the atomic property value for a digital connector.
+ */
+int intel_digital_connector_atomic_set_property(struct drm_connector *connector,
+						struct drm_connector_state *state,
+						struct drm_property *property,
+						uint64_t val)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_digital_connector_state *intel_conn_state =
+		to_intel_digital_connector_state(state);
+
+	if (property == dev_priv->force_audio_property) {
+		intel_conn_state->force_audio = val;
+		return 0;
 	}
 
+	if (property == dev_priv->broadcast_rgb_property) {
+		intel_conn_state->broadcast_rgb = val;
+		return 0;
+	}
+
+	DRM_DEBUG_ATOMIC("Unknown property %s\n", property->name);
 	return -EINVAL;
 }
 
-/*
+int intel_digital_connector_atomic_check(struct drm_connector *conn,
+					 struct drm_connector_state *new_state)
+{
+	struct intel_digital_connector_state *new_conn_state =
+		to_intel_digital_connector_state(new_state);
+	struct drm_connector_state *old_state =
+		drm_atomic_get_old_connector_state(new_state->state, conn);
+	struct intel_digital_connector_state *old_conn_state =
+		to_intel_digital_connector_state(old_state);
+	struct drm_crtc_state *crtc_state;
+
+	if (!new_state->crtc)
+		return 0;
+
+	crtc_state = drm_atomic_get_new_crtc_state(new_state->state, new_state->crtc);
+
+	/*
+	 * These properties are handled by fastset, and might not end
+	 * up in a modeset.
+	 */
+	if (new_conn_state->force_audio != old_conn_state->force_audio ||
+	    new_conn_state->broadcast_rgb != old_conn_state->broadcast_rgb ||
+	    new_conn_state->base.picture_aspect_ratio != old_conn_state->base.picture_aspect_ratio ||
+	    new_conn_state->base.scaling_mode != old_conn_state->base.scaling_mode)
+		crtc_state->mode_changed = true;
+
+	return 0;
+}
+
+/**
+ * intel_digital_connector_duplicate_state - duplicate connector state
+ * @connector: digital connector
+ *
+ * Allocates and returns a copy of the connector state (both common and
+ * digital connector specific) for the specified connector.
+ *
+ * Returns: The newly allocated connector state, or NULL on failure.
+ */
+struct drm_connector_state *
+intel_digital_connector_duplicate_state(struct drm_connector *connector)
+{
+	struct intel_digital_connector_state *state;
+
+	state = kmemdup(connector->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_connector_duplicate_state(connector, &state->base);
+	return &state->base;
+}
+
+/**
  * intel_crtc_duplicate_state - duplicate crtc state
  * @crtc: drm crtc
  *
@@ -248,7 +325,7 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv,
 		}
 
 		/* set scaler mode */
-		if (IS_GEMINILAKE(dev_priv)) {
+		if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 			scaler_state->scalers[*scaler_id].mode = 0;
 		} else if (num_scalers_need == 1 && intel_crtc->pipe != PIPE_C) {
 			/*
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index cfb47293fd53..4325cb0a04f5 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -55,7 +55,7 @@ intel_create_plane_state(struct drm_plane *plane)
 		return NULL;
 
 	state->base.plane = plane;
-	state->base.rotation = DRM_ROTATE_0;
+	state->base.rotation = DRM_MODE_ROTATE_0;
 	state->ckey.flags = I915_SET_COLORKEY_NONE;
 
 	return state;
@@ -102,23 +102,7 @@ void
 intel_plane_destroy_state(struct drm_plane *plane,
 			  struct drm_plane_state *state)
 {
-	struct i915_vma *vma;
-
-	vma = fetch_and_zero(&to_intel_plane_state(state)->vma);
-
-	/*
-	 * FIXME: Normally intel_cleanup_plane_fb handles destruction of vma.
-	 * We currently don't clear all planes during driver unload, so we have
-	 * to be able to unpin vma here for now.
-	 *
-	 * Normally this can only happen during unload when kmscon is disabled
-	 * and userspace doesn't attempt to set a framebuffer at all.
-	 */
-	if (vma) {
-		mutex_lock(&plane->dev->struct_mutex);
-		intel_unpin_fb_vma(vma);
-		mutex_unlock(&plane->dev->struct_mutex);
-	}
+	WARN_ON(to_intel_plane_state(state)->vma);
 
 	drm_atomic_helper_plane_destroy_state(plane, state);
 }
@@ -178,14 +162,14 @@ int intel_plane_atomic_check_with_state(struct intel_crtc_state *crtc_state,
 
 	/* CHV ignores the mirror bit when the rotate bit is set :( */
 	if (IS_CHERRYVIEW(dev_priv) &&
-	    state->rotation & DRM_ROTATE_180 &&
-	    state->rotation & DRM_REFLECT_X) {
+	    state->rotation & DRM_MODE_ROTATE_180 &&
+	    state->rotation & DRM_MODE_REFLECT_X) {
 		DRM_DEBUG_KMS("Cannot rotate and reflect at the same time\n");
 		return -EINVAL;
 	}
 
 	intel_state->base.visible = false;
-	ret = intel_plane->check_plane(plane, crtc_state, intel_state);
+	ret = intel_plane->check_plane(intel_plane, crtc_state, intel_state);
 	if (ret)
 		return ret;
 
@@ -235,14 +219,14 @@ static void intel_plane_atomic_update(struct drm_plane *plane,
 		trace_intel_update_plane(plane,
 					 to_intel_crtc(crtc));
 
-		intel_plane->update_plane(plane,
+		intel_plane->update_plane(intel_plane,
 					  to_intel_crtc_state(crtc->state),
 					  intel_state);
 	} else {
 		trace_intel_disable_plane(plane,
 					  to_intel_crtc(crtc));
 
-		intel_plane->disable_plane(plane, crtc);
+		intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc));
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 52c207e81f41..d805b6e6fe71 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -632,20 +632,9 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
 						 (int) port, (int) pipe);
 	}
 
-	switch (intel_encoder->type) {
-	case INTEL_OUTPUT_HDMI:
-		intel_lpe_audio_notify(dev_priv, connector->eld, port, pipe,
-				       crtc_state->port_clock,
-				       false, 0);
-		break;
-	case INTEL_OUTPUT_DP:
-		intel_lpe_audio_notify(dev_priv, connector->eld, port, pipe,
-				       adjusted_mode->crtc_clock,
-				       true, crtc_state->port_clock);
-		break;
-	default:
-		break;
-	}
+	intel_lpe_audio_notify(dev_priv, pipe, port, connector->eld,
+			       crtc_state->port_clock,
+			       intel_encoder->type == INTEL_OUTPUT_DP);
 }
 
 /**
@@ -680,7 +669,7 @@ void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
 						 (int) port, (int) pipe);
 	}
 
-	intel_lpe_audio_notify(dev_priv, NULL, port, pipe, 0, false, 0);
+	intel_lpe_audio_notify(dev_priv, pipe, port, NULL, 0, false);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 9ccbf26124c6..4e00e5cb9fa1 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -64,10 +64,12 @@ static unsigned long wait_timeout(void)
 
 static noinline void missed_breadcrumb(struct intel_engine_cs *engine)
 {
-	DRM_DEBUG_DRIVER("%s missed breadcrumb at %pF, irq posted? %s\n",
+	DRM_DEBUG_DRIVER("%s missed breadcrumb at %pF, irq posted? %s, current seqno=%x, last=%x\n",
 			 engine->name, __builtin_return_address(0),
 			 yesno(test_bit(ENGINE_IRQ_BREADCRUMB,
-					&engine->irq_posted)));
+					&engine->irq_posted)),
+			 intel_engine_get_seqno(engine),
+			 intel_engine_last_submit(engine));
 
 	set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
 }
@@ -232,7 +234,7 @@ static void enable_fake_irq(struct intel_breadcrumbs *b)
 		mod_timer(&b->hangcheck, wait_timeout());
 }
 
-static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
+static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 {
 	struct intel_engine_cs *engine =
 		container_of(b, struct intel_engine_cs, breadcrumbs);
@@ -240,7 +242,7 @@ static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 
 	lockdep_assert_held(&b->irq_lock);
 	if (b->irq_armed)
-		return;
+		return false;
 
 	/* The breadcrumb irq will be disarmed on the interrupt after the
 	 * waiters are signaled. This gives us a single interrupt window in
@@ -258,7 +260,7 @@ static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 		 * implementation to call intel_engine_wakeup()
 		 * itself when it wants to simulate a user interrupt,
 		 */
-		return;
+		return true;
 	}
 
 	/* Since we are waiting on a request, the GPU should be busy
@@ -276,6 +278,7 @@ static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 	}
 
 	enable_fake_irq(b);
+	return true;
 }
 
 static inline struct intel_wait *to_wait(struct rb_node *node)
@@ -327,7 +330,7 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 	struct rb_node **p, *parent, *completed;
-	bool first;
+	bool first, armed;
 	u32 seqno;
 
 	/* Insert the request into the retirement ordered list
@@ -342,6 +345,7 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 	 * removing stale elements in the tree, we may be able to reduce the
 	 * ping-pong between the old bottom-half and ourselves as first-waiter.
 	 */
+	armed = false;
 	first = true;
 	parent = NULL;
 	completed = NULL;
@@ -397,7 +401,7 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 		 * in the unlocked read of b->irq_seqno_bh in the irq handler)
 		 * and so we miss the wake up.
 		 */
-		__intel_breadcrumbs_enable_irq(b);
+		armed = __intel_breadcrumbs_enable_irq(b);
 		spin_unlock(&b->irq_lock);
 	}
 
@@ -424,20 +428,24 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 	GEM_BUG_ON(!b->irq_armed);
 	GEM_BUG_ON(rb_first(&b->waiters) != &b->irq_wait->node);
 
-	return first;
+	return armed;
 }
 
 bool intel_engine_add_wait(struct intel_engine_cs *engine,
 			   struct intel_wait *wait)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
-	bool first;
+	bool armed;
 
 	spin_lock_irq(&b->rb_lock);
-	first = __intel_engine_add_wait(engine, wait);
+	armed = __intel_engine_add_wait(engine, wait);
 	spin_unlock_irq(&b->rb_lock);
+	if (armed)
+		return armed;
 
-	return first;
+	/* Make the caller recheck if its request has already started. */
+	return i915_seqno_passed(intel_engine_get_seqno(engine),
+				 wait->seqno - 1);
 }
 
 static inline bool chain_wakeup(struct rb_node *rb, int priority)
@@ -665,12 +673,11 @@ static int intel_breadcrumbs_signaler(void *arg)
 	return 0;
 }
 
-void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
+void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
+				   bool wakeup)
 {
 	struct intel_engine_cs *engine = request->engine;
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
-	struct rb_node *parent, **p;
-	bool first, wakeup;
 	u32 seqno;
 
 	/* Note that we may be called from an interrupt handler on another
@@ -703,29 +710,38 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
 	 * If we are the oldest waiter, enable the irq (after which we
 	 * must double check that the seqno did not complete).
 	 */
-	wakeup = __intel_engine_add_wait(engine, &request->signaling.wait);
+	wakeup &= __intel_engine_add_wait(engine, &request->signaling.wait);
 
-	/* Now insert ourselves into the retirement ordered list of signals
-	 * on this engine. We track the oldest seqno as that will be the
-	 * first signal to complete.
-	 */
-	parent = NULL;
-	first = true;
-	p = &b->signals.rb_node;
-	while (*p) {
-		parent = *p;
-		if (i915_seqno_passed(seqno,
-				      to_signaler(parent)->signaling.wait.seqno)) {
-			p = &parent->rb_right;
-			first = false;
-		} else {
-			p = &parent->rb_left;
+	if (!__i915_gem_request_completed(request, seqno)) {
+		struct rb_node *parent, **p;
+		bool first;
+
+		/* Now insert ourselves into the retirement ordered list of
+		 * signals on this engine. We track the oldest seqno as that
+		 * will be the first signal to complete.
+		 */
+		parent = NULL;
+		first = true;
+		p = &b->signals.rb_node;
+		while (*p) {
+			parent = *p;
+			if (i915_seqno_passed(seqno,
+					      to_signaler(parent)->signaling.wait.seqno)) {
+				p = &parent->rb_right;
+				first = false;
+			} else {
+				p = &parent->rb_left;
+			}
 		}
+		rb_link_node(&request->signaling.node, parent, p);
+		rb_insert_color(&request->signaling.node, &b->signals);
+		if (first)
+			rcu_assign_pointer(b->first_signal, request);
+	} else {
+		__intel_engine_remove_wait(engine, &request->signaling.wait);
+		i915_gem_request_put(request);
+		wakeup = false;
 	}
-	rb_link_node(&request->signaling.node, parent, p);
-	rb_insert_color(&request->signaling.node, &b->signals);
-	if (first)
-		rcu_assign_pointer(b->first_signal, request);
 
 	spin_unlock(&b->rb_lock);
 
diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c
index f29a226e24d8..b8914db7d2e1 100644
--- a/drivers/gpu/drm/i915/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/intel_cdclk.c
@@ -1071,9 +1071,15 @@ static int bxt_calc_cdclk(int max_pixclk)
 
 static int glk_calc_cdclk(int max_pixclk)
 {
-	if (max_pixclk > 2 * 158400)
+	/*
+	 * FIXME: Avoid using a pixel clock that is more than 99% of the cdclk
+	 * as a temporary workaround. Use a higher cdclk instead. (Note that
+	 * intel_compute_max_dotclk() limits the max pixel clock to 99% of max
+	 * cdclk.)
+	 */
+	if (max_pixclk > DIV_ROUND_UP(2 * 158400 * 99, 100))
 		return 316800;
-	else if (max_pixclk > 2 * 79200)
+	else if (max_pixclk > DIV_ROUND_UP(2 * 79200 * 99, 100))
 		return 158400;
 	else
 		return 79200;
@@ -1394,6 +1400,280 @@ void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
 	bxt_set_cdclk(dev_priv, &cdclk_state);
 }
 
+static int cnl_calc_cdclk(int max_pixclk)
+{
+	if (max_pixclk > 336000)
+		return 528000;
+	else if (max_pixclk > 168000)
+		return 336000;
+	else
+		return 168000;
+}
+
+static void cnl_cdclk_pll_update(struct drm_i915_private *dev_priv,
+				 struct intel_cdclk_state *cdclk_state)
+{
+	u32 val;
+
+	if (I915_READ(SKL_DSSM) & CNL_DSSM_CDCLK_PLL_REFCLK_24MHz)
+		cdclk_state->ref = 24000;
+	else
+		cdclk_state->ref = 19200;
+
+	cdclk_state->vco = 0;
+
+	val = I915_READ(BXT_DE_PLL_ENABLE);
+	if ((val & BXT_DE_PLL_PLL_ENABLE) == 0)
+		return;
+
+	if (WARN_ON((val & BXT_DE_PLL_LOCK) == 0))
+		return;
+
+	cdclk_state->vco = (val & CNL_CDCLK_PLL_RATIO_MASK) * cdclk_state->ref;
+}
+
+static void cnl_get_cdclk(struct drm_i915_private *dev_priv,
+			 struct intel_cdclk_state *cdclk_state)
+{
+	u32 divider;
+	int div;
+
+	cnl_cdclk_pll_update(dev_priv, cdclk_state);
+
+	cdclk_state->cdclk = cdclk_state->ref;
+
+	if (cdclk_state->vco == 0)
+		return;
+
+	divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
+
+	switch (divider) {
+	case BXT_CDCLK_CD2X_DIV_SEL_1:
+		div = 2;
+		break;
+	case BXT_CDCLK_CD2X_DIV_SEL_2:
+		div = 4;
+		break;
+	default:
+		MISSING_CASE(divider);
+		return;
+	}
+
+	cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div);
+}
+
+static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
+{
+	u32 val;
+
+	val = I915_READ(BXT_DE_PLL_ENABLE);
+	val &= ~BXT_DE_PLL_PLL_ENABLE;
+	I915_WRITE(BXT_DE_PLL_ENABLE, val);
+
+	/* Timeout 200us */
+	if (wait_for((I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) == 0, 1))
+		DRM_ERROR("timout waiting for CDCLK PLL unlock\n");
+
+	dev_priv->cdclk.hw.vco = 0;
+}
+
+static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
+{
+	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
+	u32 val;
+
+	val = CNL_CDCLK_PLL_RATIO(ratio);
+	I915_WRITE(BXT_DE_PLL_ENABLE, val);
+
+	val |= BXT_DE_PLL_PLL_ENABLE;
+	I915_WRITE(BXT_DE_PLL_ENABLE, val);
+
+	/* Timeout 200us */
+	if (wait_for((I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) != 0, 1))
+		DRM_ERROR("timout waiting for CDCLK PLL lock\n");
+
+	dev_priv->cdclk.hw.vco = vco;
+}
+
+static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
+			  const struct intel_cdclk_state *cdclk_state)
+{
+	int cdclk = cdclk_state->cdclk;
+	int vco = cdclk_state->vco;
+	u32 val, divider, pcu_ack;
+	int ret;
+
+	mutex_lock(&dev_priv->rps.hw_lock);
+	ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
+				SKL_CDCLK_PREPARE_FOR_CHANGE,
+				SKL_CDCLK_READY_FOR_CHANGE,
+				SKL_CDCLK_READY_FOR_CHANGE, 3);
+	mutex_unlock(&dev_priv->rps.hw_lock);
+	if (ret) {
+		DRM_ERROR("Failed to inform PCU about cdclk change (%d)\n",
+			  ret);
+		return;
+	}
+
+	/* cdclk = vco / 2 / div{1,2} */
+	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
+	case 4:
+		divider = BXT_CDCLK_CD2X_DIV_SEL_2;
+		break;
+	case 2:
+		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+		break;
+	default:
+		WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
+		WARN_ON(vco != 0);
+
+		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+		break;
+	}
+
+	switch (cdclk) {
+	case 528000:
+		pcu_ack = 2;
+		break;
+	case 336000:
+		pcu_ack = 1;
+		break;
+	case 168000:
+	default:
+		pcu_ack = 0;
+		break;
+	}
+
+	if (dev_priv->cdclk.hw.vco != 0 &&
+	    dev_priv->cdclk.hw.vco != vco)
+		cnl_cdclk_pll_disable(dev_priv);
+
+	if (dev_priv->cdclk.hw.vco != vco)
+		cnl_cdclk_pll_enable(dev_priv, vco);
+
+	val = divider | skl_cdclk_decimal(cdclk);
+	/*
+	 * FIXME if only the cd2x divider needs changing, it could be done
+	 * without shutting off the pipe (if only one pipe is active).
+	 */
+	val |= BXT_CDCLK_CD2X_PIPE_NONE;
+	I915_WRITE(CDCLK_CTL, val);
+
+	/* inform PCU of the change */
+	mutex_lock(&dev_priv->rps.hw_lock);
+	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
+	mutex_unlock(&dev_priv->rps.hw_lock);
+
+	intel_update_cdclk(dev_priv);
+}
+
+static int cnl_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
+{
+	int ratio;
+
+	if (cdclk == dev_priv->cdclk.hw.ref)
+		return 0;
+
+	switch (cdclk) {
+	default:
+		MISSING_CASE(cdclk);
+	case 168000:
+	case 336000:
+		ratio = dev_priv->cdclk.hw.ref == 19200 ? 35 : 28;
+		break;
+	case 528000:
+		ratio = dev_priv->cdclk.hw.ref == 19200 ? 55 : 44;
+		break;
+	}
+
+	return dev_priv->cdclk.hw.ref * ratio;
+}
+
+static void cnl_sanitize_cdclk(struct drm_i915_private *dev_priv)
+{
+	u32 cdctl, expected;
+
+	intel_update_cdclk(dev_priv);
+
+	if (dev_priv->cdclk.hw.vco == 0 ||
+	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
+		goto sanitize;
+
+	/* DPLL okay; verify the cdclock
+	 *
+	 * Some BIOS versions leave an incorrect decimal frequency value and
+	 * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
+	 * so sanitize this register.
+	 */
+	cdctl = I915_READ(CDCLK_CTL);
+	/*
+	 * Let's ignore the pipe field, since BIOS could have configured the
+	 * dividers both synching to an active pipe, or asynchronously
+	 * (PIPE_NONE).
+	 */
+	cdctl &= ~BXT_CDCLK_CD2X_PIPE_NONE;
+
+	expected = (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) |
+		   skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
+
+	if (cdctl == expected)
+		/* All well; nothing to sanitize */
+		return;
+
+sanitize:
+	DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n");
+
+	/* force cdclk programming */
+	dev_priv->cdclk.hw.cdclk = 0;
+
+	/* force full PLL disable + enable */
+	dev_priv->cdclk.hw.vco = -1;
+}
+
+/**
+ * cnl_init_cdclk - Initialize CDCLK on CNL
+ * @dev_priv: i915 device
+ *
+ * Initialize CDCLK for CNL. This is generally
+ * done only during the display core initialization sequence,
+ * after which the DMC will take care of turning CDCLK off/on
+ * as needed.
+ */
+void cnl_init_cdclk(struct drm_i915_private *dev_priv)
+{
+	struct intel_cdclk_state cdclk_state;
+
+	cnl_sanitize_cdclk(dev_priv);
+
+	if (dev_priv->cdclk.hw.cdclk != 0 &&
+	    dev_priv->cdclk.hw.vco != 0)
+		return;
+
+	cdclk_state = dev_priv->cdclk.hw;
+
+	cdclk_state.cdclk = cnl_calc_cdclk(0);
+	cdclk_state.vco = cnl_cdclk_pll_vco(dev_priv, cdclk_state.cdclk);
+
+	cnl_set_cdclk(dev_priv, &cdclk_state);
+}
+
+/**
+ * cnl_uninit_cdclk - Uninitialize CDCLK on CNL
+ * @dev_priv: i915 device
+ *
+ * Uninitialize CDCLK for CNL. This is done only
+ * during the display core uninitialization sequence.
+ */
+void cnl_uninit_cdclk(struct drm_i915_private *dev_priv)
+{
+	struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw;
+
+	cdclk_state.cdclk = cdclk_state.ref;
+	cdclk_state.vco = 0;
+
+	cnl_set_cdclk(dev_priv, &cdclk_state);
+}
+
 /**
  * intel_cdclk_state_compare - Determine if two CDCLK states differ
  * @a: first CDCLK state
@@ -1452,7 +1732,9 @@ static int bdw_adjust_min_pipe_pixel_rate(struct intel_crtc_state *crtc_state,
 	    crtc_state->has_audio &&
 	    crtc_state->port_clock >= 540000 &&
 	    crtc_state->lane_count == 4) {
-		if (IS_GEMINILAKE(dev_priv))
+		if (IS_CANNONLAKE(dev_priv))
+			pixel_rate = max(316800, pixel_rate);
+		else if (IS_GEMINILAKE(dev_priv))
 			pixel_rate = max(2 * 316800, pixel_rate);
 		else
 			pixel_rate = max(432000, pixel_rate);
@@ -1498,7 +1780,7 @@ static int intel_max_pixel_rate(struct drm_atomic_state *state)
 
 		pixel_rate = crtc_state->pixel_rate;
 
-		if (IS_BROADWELL(dev_priv) || IS_GEN9(dev_priv))
+		if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9)
 			pixel_rate =
 				bdw_adjust_min_pipe_pixel_rate(crtc_state,
 							       pixel_rate);
@@ -1659,12 +1941,50 @@ static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 	return 0;
 }
 
+static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state)
+{
+	struct drm_i915_private *dev_priv = to_i915(state->dev);
+	struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(state);
+	int max_pixclk = intel_max_pixel_rate(state);
+	int cdclk, vco;
+
+	cdclk = cnl_calc_cdclk(max_pixclk);
+	vco = cnl_cdclk_pll_vco(dev_priv, cdclk);
+
+	if (cdclk > dev_priv->max_cdclk_freq) {
+		DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
+			      cdclk, dev_priv->max_cdclk_freq);
+		return -EINVAL;
+	}
+
+	intel_state->cdclk.logical.vco = vco;
+	intel_state->cdclk.logical.cdclk = cdclk;
+
+	if (!intel_state->active_crtcs) {
+		cdclk = cnl_calc_cdclk(0);
+		vco = cnl_cdclk_pll_vco(dev_priv, cdclk);
+
+		intel_state->cdclk.actual.vco = vco;
+		intel_state->cdclk.actual.cdclk = cdclk;
+	} else {
+		intel_state->cdclk.actual =
+			intel_state->cdclk.logical;
+	}
+
+	return 0;
+}
+
 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
 {
 	int max_cdclk_freq = dev_priv->max_cdclk_freq;
 
 	if (IS_GEMINILAKE(dev_priv))
-		return 2 * max_cdclk_freq;
+		/*
+		 * FIXME: Limiting to 99% as a temporary workaround. See
+		 * glk_calc_cdclk() for details.
+		 */
+		return 2 * max_cdclk_freq * 99 / 100;
 	else if (INTEL_INFO(dev_priv)->gen >= 9 ||
 		 IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		return max_cdclk_freq;
@@ -1686,7 +2006,9 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
  */
 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 {
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_CANNONLAKE(dev_priv)) {
+		dev_priv->max_cdclk_freq = 528000;
+	} else if (IS_GEN9_BC(dev_priv)) {
 		u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
 		int max_cdclk, vco;
 
@@ -1770,6 +2092,30 @@ void intel_update_cdclk(struct drm_i915_private *dev_priv)
 			   DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
 }
 
+static int cnp_rawclk(struct drm_i915_private *dev_priv)
+{
+	u32 rawclk;
+	int divider, fraction;
+
+	if (I915_READ(SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
+		/* 24 MHz */
+		divider = 24000;
+		fraction = 0;
+	} else {
+		/* 19.2 MHz */
+		divider = 19000;
+		fraction = 200;
+	}
+
+	rawclk = CNP_RAWCLK_DIV((divider / 1000) - 1);
+	if (fraction)
+		rawclk |= CNP_RAWCLK_FRAC(DIV_ROUND_CLOSEST(1000,
+							    fraction) - 1);
+
+	I915_WRITE(PCH_RAWCLK_FREQ, rawclk);
+	return divider + fraction;
+}
+
 static int pch_rawclk(struct drm_i915_private *dev_priv)
 {
 	return (I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
@@ -1817,7 +2163,10 @@ static int g4x_hrawclk(struct drm_i915_private *dev_priv)
  */
 void intel_update_rawclk(struct drm_i915_private *dev_priv)
 {
-	if (HAS_PCH_SPLIT(dev_priv))
+
+	if (HAS_PCH_CNP(dev_priv))
+		dev_priv->rawclk_freq = cnp_rawclk(dev_priv);
+	else if (HAS_PCH_SPLIT(dev_priv))
 		dev_priv->rawclk_freq = pch_rawclk(dev_priv);
 	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		dev_priv->rawclk_freq = vlv_hrawclk(dev_priv);
@@ -1856,9 +2205,15 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
 		dev_priv->display.set_cdclk = skl_set_cdclk;
 		dev_priv->display.modeset_calc_cdclk =
 			skl_modeset_calc_cdclk;
+	} else if (IS_CANNONLAKE(dev_priv)) {
+		dev_priv->display.set_cdclk = cnl_set_cdclk;
+		dev_priv->display.modeset_calc_cdclk =
+			cnl_modeset_calc_cdclk;
 	}
 
-	if (IS_GEN9_BC(dev_priv))
+	if (IS_CANNONLAKE(dev_priv))
+		dev_priv->display.get_cdclk = cnl_get_cdclk;
+	else if (IS_GEN9_BC(dev_priv))
 		dev_priv->display.get_cdclk = skl_get_cdclk;
 	else if (IS_GEN9_LP(dev_priv))
 		dev_priv->display.get_cdclk = bxt_get_cdclk;
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 2797bf37c3ac..84a1f5e85153 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -777,13 +777,6 @@ out:
 	return ret;
 }
 
-static int intel_crt_set_property(struct drm_connector *connector,
-				  struct drm_property *property,
-				  uint64_t value)
-{
-	return 0;
-}
-
 void intel_crt_reset(struct drm_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
@@ -814,10 +807,9 @@ static const struct drm_connector_funcs intel_crt_connector_funcs = {
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_crt_destroy,
-	.set_property = intel_crt_set_property,
+	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_get_property = intel_connector_atomic_get_property,
 };
 
 static const struct drm_connector_helper_funcs intel_crt_connector_helper_funcs = {
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index 1575bde0cf90..965988f79a55 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -37,6 +37,9 @@
 #define I915_CSR_GLK "i915/glk_dmc_ver1_04.bin"
 #define GLK_CSR_VERSION_REQUIRED	CSR_VERSION(1, 4)
 
+#define I915_CSR_CNL "i915/cnl_dmc_ver1_04.bin"
+#define CNL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 4)
+
 #define I915_CSR_KBL "i915/kbl_dmc_ver1_01.bin"
 MODULE_FIRMWARE(I915_CSR_KBL);
 #define KBL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 1)
@@ -238,7 +241,7 @@ void intel_csr_load_program(struct drm_i915_private *dev_priv)
 	u32 *payload = dev_priv->csr.dmc_payload;
 	uint32_t i, fw_size;
 
-	if (!IS_GEN9(dev_priv)) {
+	if (!HAS_CSR(dev_priv)) {
 		DRM_ERROR("No CSR support available for this platform\n");
 		return;
 	}
@@ -289,9 +292,11 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv,
 
 	csr->version = css_header->version;
 
-	if (IS_GEMINILAKE(dev_priv)) {
+	if (IS_CANNONLAKE(dev_priv)) {
+		required_version = CNL_CSR_VERSION_REQUIRED;
+	} else if (IS_GEMINILAKE(dev_priv)) {
 		required_version = GLK_CSR_VERSION_REQUIRED;
-	} else if (IS_KABYLAKE(dev_priv)) {
+	} else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
 		required_version = KBL_CSR_VERSION_REQUIRED;
 	} else if (IS_SKYLAKE(dev_priv)) {
 		required_version = SKL_CSR_VERSION_REQUIRED;
@@ -438,9 +443,11 @@ void intel_csr_ucode_init(struct drm_i915_private *dev_priv)
 	if (!HAS_CSR(dev_priv))
 		return;
 
-	if (IS_GEMINILAKE(dev_priv))
+	if (IS_CANNONLAKE(dev_priv))
+		csr->fw_path = I915_CSR_CNL;
+	else if (IS_GEMINILAKE(dev_priv))
 		csr->fw_path = I915_CSR_GLK;
-	else if (IS_KABYLAKE(dev_priv))
+	else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
 		csr->fw_path = I915_CSR_KBL;
 	else if (IS_SKYLAKE(dev_priv))
 		csr->fw_path = I915_CSR_SKL;
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 0914ad96a71b..80e96f1f49d2 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -353,6 +353,146 @@ static const struct bxt_ddi_buf_trans bxt_ddi_translations_hdmi[] = {
 	{ 154, 0x9A, 1, 128, true },	/* 9:	1200		0   */
 };
 
+struct cnl_ddi_buf_trans {
+	u32 dw2_swing_sel;
+	u32 dw7_n_scalar;
+	u32 dw4_cursor_coeff;
+	u32 dw4_post_cursor_2;
+	u32 dw4_post_cursor_1;
+};
+
+/* Voltage Swing Programming for VccIO 0.85V for DP */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_dp_0_85V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x5D, 0x3F, 0x00, 0x00 },	/* 350   350      0.0   */
+	{ 0xA, 0x6A, 0x38, 0x00, 0x07 },	/* 350   500      3.1   */
+	{ 0xB, 0x7A, 0x32, 0x00, 0x0D },	/* 350   700      6.0   */
+	{ 0x6, 0x7C, 0x2D, 0x00, 0x12 },	/* 350   900      8.2   */
+	{ 0xA, 0x69, 0x3F, 0x00, 0x00 },	/* 500   500      0.0   */
+	{ 0xB, 0x7A, 0x36, 0x00, 0x09 },	/* 500   700      2.9   */
+	{ 0x6, 0x7C, 0x30, 0x00, 0x0F },	/* 500   900      5.1   */
+	{ 0xB, 0x7D, 0x3C, 0x00, 0x03 },	/* 650   725      0.9   */
+	{ 0x6, 0x7C, 0x34, 0x00, 0x0B },	/* 600   900      3.5   */
+	{ 0x6, 0x7B, 0x3F, 0x00, 0x00 },	/* 900   900      0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 0.85V for HDMI */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_hdmi_0_85V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x60, 0x3F, 0x00, 0x00 },	/* 450   450      0.0   */
+	{ 0xB, 0x73, 0x36, 0x00, 0x09 },	/* 450   650      3.2   */
+	{ 0x6, 0x7F, 0x31, 0x00, 0x0E },	/* 450   850      5.5   */
+	{ 0xB, 0x73, 0x3F, 0x00, 0x00 },	/* 650   650      0.0   */
+	{ 0x6, 0x7F, 0x37, 0x00, 0x08 },	/* 650   850      2.3   */
+	{ 0x6, 0x7F, 0x3F, 0x00, 0x00 },	/* 850   850      0.0   */
+	{ 0x6, 0x7F, 0x35, 0x00, 0x0A },	/* 600   850      3.0   */
+};
+
+/* Voltage Swing Programming for VccIO 0.85V for eDP */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_0_85V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x66, 0x3A, 0x00, 0x05 },	/* 384   500      2.3   */
+	{ 0x0, 0x7F, 0x38, 0x00, 0x07 },	/* 153   200      2.3   */
+	{ 0x8, 0x7F, 0x38, 0x00, 0x07 },	/* 192   250      2.3   */
+	{ 0x1, 0x7F, 0x38, 0x00, 0x07 },	/* 230   300      2.3   */
+	{ 0x9, 0x7F, 0x38, 0x00, 0x07 },	/* 269   350      2.3   */
+	{ 0xA, 0x66, 0x3C, 0x00, 0x03 },	/* 446   500      1.0   */
+	{ 0xB, 0x70, 0x3C, 0x00, 0x03 },	/* 460   600      2.3   */
+	{ 0xC, 0x75, 0x3C, 0x00, 0x03 },	/* 537   700      2.3   */
+	{ 0x2, 0x7F, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 0.95V for DP */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_dp_0_95V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x5D, 0x3F, 0x00, 0x00 },	/* 350   350      0.0   */
+	{ 0xA, 0x6A, 0x38, 0x00, 0x07 },	/* 350   500      3.1   */
+	{ 0xB, 0x7A, 0x32, 0x00, 0x0D },	/* 350   700      6.0   */
+	{ 0x6, 0x7C, 0x2D, 0x00, 0x12 },	/* 350   900      8.2   */
+	{ 0xA, 0x69, 0x3F, 0x00, 0x00 },	/* 500   500      0.0   */
+	{ 0xB, 0x7A, 0x36, 0x00, 0x09 },	/* 500   700      2.9   */
+	{ 0x6, 0x7C, 0x30, 0x00, 0x0F },	/* 500   900      5.1   */
+	{ 0xB, 0x7D, 0x3C, 0x00, 0x03 },	/* 650   725      0.9   */
+	{ 0x6, 0x7C, 0x34, 0x00, 0x0B },	/* 600   900      3.5   */
+	{ 0x6, 0x7B, 0x3F, 0x00, 0x00 },	/* 900   900      0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 0.95V for HDMI */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_hdmi_0_95V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x5C, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
+	{ 0xB, 0x69, 0x37, 0x00, 0x08 },	/* 400   600      3.5   */
+	{ 0x5, 0x76, 0x31, 0x00, 0x0E },	/* 400   800      6.0   */
+	{ 0xA, 0x5E, 0x3F, 0x00, 0x00 },	/* 450   450      0.0   */
+	{ 0xB, 0x69, 0x3F, 0x00, 0x00 },	/* 600   600      0.0   */
+	{ 0xB, 0x79, 0x35, 0x00, 0x0A },	/* 600   850      3.0   */
+	{ 0x6, 0x7D, 0x32, 0x00, 0x0D },	/* 600   1000     4.4   */
+	{ 0x5, 0x76, 0x3F, 0x00, 0x00 },	/* 800   800      0.0   */
+	{ 0x6, 0x7D, 0x39, 0x00, 0x06 },	/* 800   1000     1.9   */
+	{ 0x6, 0x7F, 0x39, 0x00, 0x06 },	/* 850   1050     1.8   */
+	{ 0x6, 0x7F, 0x3F, 0x00, 0x00 },	/* 1050  1050     0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 0.95V for eDP */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_0_95V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x61, 0x3A, 0x00, 0x05 },	/* 384   500      2.3   */
+	{ 0x0, 0x7F, 0x38, 0x00, 0x07 },	/* 153   200      2.3   */
+	{ 0x8, 0x7F, 0x38, 0x00, 0x07 },	/* 192   250      2.3   */
+	{ 0x1, 0x7F, 0x38, 0x00, 0x07 },	/* 230   300      2.3   */
+	{ 0x9, 0x7F, 0x38, 0x00, 0x07 },	/* 269   350      2.3   */
+	{ 0xA, 0x61, 0x3C, 0x00, 0x03 },	/* 446   500      1.0   */
+	{ 0xB, 0x68, 0x39, 0x00, 0x06 },	/* 460   600      2.3   */
+	{ 0xC, 0x6E, 0x39, 0x00, 0x06 },	/* 537   700      2.3   */
+	{ 0x4, 0x7F, 0x3A, 0x00, 0x05 },	/* 460   600      2.3   */
+	{ 0x2, 0x7F, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 1.05V for DP */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_dp_1_05V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x58, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
+	{ 0xB, 0x64, 0x37, 0x00, 0x08 },	/* 400   600      3.5   */
+	{ 0x5, 0x70, 0x31, 0x00, 0x0E },	/* 400   800      6.0   */
+	{ 0x6, 0x7F, 0x2C, 0x00, 0x13 },	/* 400   1050     8.4   */
+	{ 0xB, 0x64, 0x3F, 0x00, 0x00 },	/* 600   600      0.0   */
+	{ 0x5, 0x73, 0x35, 0x00, 0x0A },	/* 600   850      3.0   */
+	{ 0x6, 0x7F, 0x30, 0x00, 0x0F },	/* 550   1050     5.6   */
+	{ 0x5, 0x76, 0x3E, 0x00, 0x01 },	/* 850   900      0.5   */
+	{ 0x6, 0x7F, 0x36, 0x00, 0x09 },	/* 750   1050     2.9   */
+	{ 0x6, 0x7F, 0x3F, 0x00, 0x00 },	/* 1050  1050     0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 1.05V for HDMI */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_hdmi_1_05V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x58, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
+	{ 0xB, 0x64, 0x37, 0x00, 0x08 },	/* 400   600      3.5   */
+	{ 0x5, 0x70, 0x31, 0x00, 0x0E },	/* 400   800      6.0   */
+	{ 0xA, 0x5B, 0x3F, 0x00, 0x00 },	/* 450   450      0.0   */
+	{ 0xB, 0x64, 0x3F, 0x00, 0x00 },	/* 600   600      0.0   */
+	{ 0x5, 0x73, 0x35, 0x00, 0x0A },	/* 600   850      3.0   */
+	{ 0x6, 0x7C, 0x32, 0x00, 0x0D },	/* 600   1000     4.4   */
+	{ 0x5, 0x70, 0x3F, 0x00, 0x00 },	/* 800   800      0.0   */
+	{ 0x6, 0x7C, 0x39, 0x00, 0x06 },	/* 800   1000     1.9   */
+	{ 0x6, 0x7F, 0x39, 0x00, 0x06 },	/* 850   1050     1.8   */
+	{ 0x6, 0x7F, 0x3F, 0x00, 0x00 },	/* 1050  1050     0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 1.05V for eDP */
+static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_1_05V[] = {
+						/* NT mV Trans mV db    */
+	{ 0xA, 0x5E, 0x3A, 0x00, 0x05 },	/* 384   500      2.3   */
+	{ 0x0, 0x7F, 0x38, 0x00, 0x07 },	/* 153   200      2.3   */
+	{ 0x8, 0x7F, 0x38, 0x00, 0x07 },	/* 192   250      2.3   */
+	{ 0x1, 0x7F, 0x38, 0x00, 0x07 },	/* 230   300      2.3   */
+	{ 0x9, 0x7F, 0x38, 0x00, 0x07 },	/* 269   350      2.3   */
+	{ 0xA, 0x5E, 0x3C, 0x00, 0x03 },	/* 446   500      1.0   */
+	{ 0xB, 0x64, 0x39, 0x00, 0x06 },	/* 460   600      2.3   */
+	{ 0xE, 0x6A, 0x39, 0x00, 0x06 },	/* 537   700      2.3   */
+	{ 0x2, 0x7F, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
+};
+
 enum port intel_ddi_get_encoder_port(struct intel_encoder *encoder)
 {
 	switch (encoder->type) {
@@ -404,7 +544,7 @@ kbl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
 	if (IS_KBL_ULX(dev_priv)) {
 		*n_entries = ARRAY_SIZE(kbl_y_ddi_translations_dp);
 		return kbl_y_ddi_translations_dp;
-	} else if (IS_KBL_ULT(dev_priv)) {
+	} else if (IS_KBL_ULT(dev_priv) || IS_CFL_ULT(dev_priv)) {
 		*n_entries = ARRAY_SIZE(kbl_u_ddi_translations_dp);
 		return kbl_u_ddi_translations_dp;
 	} else {
@@ -420,7 +560,8 @@ skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 		if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv)) {
 			*n_entries = ARRAY_SIZE(skl_y_ddi_translations_edp);
 			return skl_y_ddi_translations_edp;
-		} else if (IS_SKL_ULT(dev_priv) || IS_KBL_ULT(dev_priv)) {
+		} else if (IS_SKL_ULT(dev_priv) || IS_KBL_ULT(dev_priv) ||
+			   IS_CFL_ULT(dev_priv)) {
 			*n_entries = ARRAY_SIZE(skl_u_ddi_translations_edp);
 			return skl_u_ddi_translations_edp;
 		} else {
@@ -429,7 +570,7 @@ skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 		}
 	}
 
-	if (IS_KABYLAKE(dev_priv))
+	if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
 		return kbl_get_buf_trans_dp(dev_priv, n_entries);
 	else
 		return skl_get_buf_trans_dp(dev_priv, n_entries);
@@ -485,7 +626,7 @@ static const struct ddi_buf_trans *
 intel_ddi_get_buf_trans_dp(struct drm_i915_private *dev_priv,
 			   int *n_entries)
 {
-	if (IS_KABYLAKE(dev_priv)) {
+	if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
 		return kbl_get_buf_trans_dp(dev_priv, n_entries);
 	} else if (IS_SKYLAKE(dev_priv)) {
 		return skl_get_buf_trans_dp(dev_priv, n_entries);
@@ -505,7 +646,7 @@ static const struct ddi_buf_trans *
 intel_ddi_get_buf_trans_edp(struct drm_i915_private *dev_priv,
 			    int *n_entries)
 {
-	if (IS_KABYLAKE(dev_priv) || IS_SKYLAKE(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv)) {
 		return skl_get_buf_trans_edp(dev_priv, n_entries);
 	} else if (IS_BROADWELL(dev_priv)) {
 		return bdw_get_buf_trans_edp(dev_priv, n_entries);
@@ -1478,7 +1619,7 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, u32 level)
 		if (dp_iboost) {
 			iboost = dp_iboost;
 		} else {
-			if (IS_KABYLAKE(dev_priv))
+			if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
 				ddi_translations = kbl_get_buf_trans_dp(dev_priv,
 									&n_entries);
 			else
@@ -1580,6 +1721,207 @@ u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder)
 		DP_TRAIN_VOLTAGE_SWING_MASK;
 }
 
+static const struct cnl_ddi_buf_trans *
+cnl_get_buf_trans_hdmi(struct drm_i915_private *dev_priv,
+		       u32 voltage, int *n_entries)
+{
+	if (voltage == VOLTAGE_INFO_0_85V) {
+		*n_entries = ARRAY_SIZE(cnl_ddi_translations_hdmi_0_85V);
+		return cnl_ddi_translations_hdmi_0_85V;
+	} else if (voltage == VOLTAGE_INFO_0_95V) {
+		*n_entries = ARRAY_SIZE(cnl_ddi_translations_hdmi_0_95V);
+		return cnl_ddi_translations_hdmi_0_95V;
+	} else if (voltage == VOLTAGE_INFO_1_05V) {
+		*n_entries = ARRAY_SIZE(cnl_ddi_translations_hdmi_1_05V);
+		return cnl_ddi_translations_hdmi_1_05V;
+	}
+	return NULL;
+}
+
+static const struct cnl_ddi_buf_trans *
+cnl_get_buf_trans_dp(struct drm_i915_private *dev_priv,
+		     u32 voltage, int *n_entries)
+{
+	if (voltage == VOLTAGE_INFO_0_85V) {
+		*n_entries = ARRAY_SIZE(cnl_ddi_translations_dp_0_85V);
+		return cnl_ddi_translations_dp_0_85V;
+	} else if (voltage == VOLTAGE_INFO_0_95V) {
+		*n_entries = ARRAY_SIZE(cnl_ddi_translations_dp_0_95V);
+		return cnl_ddi_translations_dp_0_95V;
+	} else if (voltage == VOLTAGE_INFO_1_05V) {
+		*n_entries = ARRAY_SIZE(cnl_ddi_translations_dp_1_05V);
+		return cnl_ddi_translations_dp_1_05V;
+	}
+	return NULL;
+}
+
+static const struct cnl_ddi_buf_trans *
+cnl_get_buf_trans_edp(struct drm_i915_private *dev_priv,
+		      u32 voltage, int *n_entries)
+{
+	if (dev_priv->vbt.edp.low_vswing) {
+		if (voltage == VOLTAGE_INFO_0_85V) {
+			*n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_0_85V);
+			return cnl_ddi_translations_dp_0_85V;
+		} else if (voltage == VOLTAGE_INFO_0_95V) {
+			*n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_0_95V);
+			return cnl_ddi_translations_edp_0_95V;
+		} else if (voltage == VOLTAGE_INFO_1_05V) {
+			*n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_1_05V);
+			return cnl_ddi_translations_edp_1_05V;
+		}
+		return NULL;
+	} else {
+		return cnl_get_buf_trans_dp(dev_priv, voltage, n_entries);
+	}
+}
+
+static void cnl_ddi_vswing_program(struct drm_i915_private *dev_priv,
+				    u32 level, enum port port, int type)
+{
+	const struct cnl_ddi_buf_trans *ddi_translations = NULL;
+	u32 n_entries, val, voltage;
+	int ln;
+
+	/*
+	 * Values for each port type are listed in
+	 * voltage swing programming tables.
+	 * Vccio voltage found in PORT_COMP_DW3.
+	 */
+	voltage = I915_READ(CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK;
+
+	if (type == INTEL_OUTPUT_HDMI) {
+		ddi_translations = cnl_get_buf_trans_hdmi(dev_priv,
+							  voltage, &n_entries);
+	} else if (type == INTEL_OUTPUT_DP) {
+		ddi_translations = cnl_get_buf_trans_dp(dev_priv,
+							voltage, &n_entries);
+	} else if (type == INTEL_OUTPUT_EDP) {
+		ddi_translations = cnl_get_buf_trans_edp(dev_priv,
+							 voltage, &n_entries);
+	}
+
+	if (ddi_translations == NULL) {
+		MISSING_CASE(voltage);
+		return;
+	}
+
+	if (level >= n_entries) {
+		DRM_DEBUG_KMS("DDI translation not found for level %d. Using %d instead.", level, n_entries - 1);
+		level = n_entries - 1;
+	}
+
+	/* Set PORT_TX_DW5 Scaling Mode Sel to 010b. */
+	val = I915_READ(CNL_PORT_TX_DW5_LN0(port));
+	val &= ~SCALING_MODE_SEL_MASK;
+	val |= SCALING_MODE_SEL(2);
+	I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val);
+
+	/* Program PORT_TX_DW2 */
+	val = I915_READ(CNL_PORT_TX_DW2_LN0(port));
+	val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK |
+		 RCOMP_SCALAR_MASK);
+	val |= SWING_SEL_UPPER(ddi_translations[level].dw2_swing_sel);
+	val |= SWING_SEL_LOWER(ddi_translations[level].dw2_swing_sel);
+	/* Rcomp scalar is fixed as 0x98 for every table entry */
+	val |= RCOMP_SCALAR(0x98);
+	I915_WRITE(CNL_PORT_TX_DW2_GRP(port), val);
+
+        /* Program PORT_TX_DW4 */
+	/* We cannot write to GRP. It would overrite individual loadgen */
+	for (ln = 0; ln < 4; ln++) {
+		val = I915_READ(CNL_PORT_TX_DW4_LN(port, ln));
+		val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
+			 CURSOR_COEFF_MASK);
+		val |= POST_CURSOR_1(ddi_translations[level].dw4_post_cursor_1);
+		val |= POST_CURSOR_2(ddi_translations[level].dw4_post_cursor_2);
+		val |= CURSOR_COEFF(ddi_translations[level].dw4_cursor_coeff);
+		I915_WRITE(CNL_PORT_TX_DW4_LN(port, ln), val);
+	}
+
+        /* Program PORT_TX_DW5 */
+	/* All DW5 values are fixed for every table entry */
+	val = I915_READ(CNL_PORT_TX_DW5_LN0(port));
+	val &= ~RTERM_SELECT_MASK;
+	val |= RTERM_SELECT(6);
+	val |= TAP3_DISABLE;
+	I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val);
+
+        /* Program PORT_TX_DW7 */
+	val = I915_READ(CNL_PORT_TX_DW7_LN0(port));
+	val &= ~N_SCALAR_MASK;
+	val |= N_SCALAR(ddi_translations[level].dw7_n_scalar);
+	I915_WRITE(CNL_PORT_TX_DW7_GRP(port), val);
+}
+
+static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder, u32 level)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	enum port port = intel_ddi_get_encoder_port(encoder);
+	int type = encoder->type;
+	int width = 0;
+	int rate = 0;
+	u32 val;
+	int ln = 0;
+
+	if ((intel_dp) && (type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP)) {
+		width = intel_dp->lane_count;
+		rate = intel_dp->link_rate;
+	} else {
+		width = 4;
+		/* Rate is always < than 6GHz for HDMI */
+	}
+
+	/*
+	 * 1. If port type is eDP or DP,
+	 * set PORT_PCS_DW1 cmnkeeper_enable to 1b,
+	 * else clear to 0b.
+	 */
+	val = I915_READ(CNL_PORT_PCS_DW1_LN0(port));
+	if (type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP)
+		val |= COMMON_KEEPER_EN;
+	else
+		val &= ~COMMON_KEEPER_EN;
+	I915_WRITE(CNL_PORT_PCS_DW1_GRP(port), val);
+
+	/* 2. Program loadgen select */
+	/*
+	 * Program PORT_TX_DW4_LN depending on Bit rate and used lanes
+	 * <= 6 GHz and 4 lanes (LN0=0, LN1=1, LN2=1, LN3=1)
+	 * <= 6 GHz and 1,2 lanes (LN0=0, LN1=1, LN2=1, LN3=0)
+	 * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0)
+	 */
+	for (ln = 0; ln <= 3; ln++) {
+		val = I915_READ(CNL_PORT_TX_DW4_LN(port, ln));
+		val &= ~LOADGEN_SELECT;
+
+		if (((rate < 600000) && (width == 4) && (ln >= 1))  ||
+		    ((rate < 600000) && (width < 4) && ((ln == 1) || (ln == 2)))) {
+			val |= LOADGEN_SELECT;
+		}
+		I915_WRITE(CNL_PORT_TX_DW4_LN(port, ln), val);
+	}
+
+	/* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */
+	val = I915_READ(CNL_PORT_CL1CM_DW5);
+	val |= SUS_CLOCK_CONFIG;
+	I915_WRITE(CNL_PORT_CL1CM_DW5, val);
+
+	/* 4. Clear training enable to change swing values */
+	val = I915_READ(CNL_PORT_TX_DW5_LN0(port));
+	val &= ~TX_TRAINING_EN;
+	I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val);
+
+	/* 5. Program swing and de-emphasis */
+	cnl_ddi_vswing_program(dev_priv, level, port, type);
+
+	/* 6. Set training enable to trigger update */
+	val = I915_READ(CNL_PORT_TX_DW5_LN0(port));
+	val |= TX_TRAINING_EN;
+	I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val);
+}
+
 static uint32_t translate_signal_level(int signal_levels)
 {
 	int i;
@@ -1612,7 +1954,11 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp)
 		skl_ddi_set_iboost(encoder, level);
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_ddi_vswing_sequence(dev_priv, level, port, encoder->type);
-
+	else if (IS_CANNONLAKE(dev_priv)) {
+		cnl_ddi_vswing_sequence(encoder, level);
+		/* DDI_BUF_CTL bits 27:24 are reserved on CNL */
+		return 0;
+	}
 	return DDI_BUF_TRANS_SELECT(level);
 }
 
@@ -1621,13 +1967,27 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = intel_ddi_get_encoder_port(encoder);
+	uint32_t val;
 
 	if (WARN_ON(!pll))
 		return;
 
-	if (IS_GEN9_BC(dev_priv)) {
-		uint32_t val;
+	if (IS_CANNONLAKE(dev_priv)) {
+		/* Configure DPCLKA_CFGCR0 to map the DPLL to the DDI. */
+		val = I915_READ(DPCLKA_CFGCR0);
+		val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->id, port);
+		I915_WRITE(DPCLKA_CFGCR0, val);
 
+		/*
+		 * Configure DPCLKA_CFGCR0 to turn on the clock for the DDI.
+		 * This step and the step before must be done with separate
+		 * register writes.
+		 */
+		val = I915_READ(DPCLKA_CFGCR0);
+		val &= ~(DPCLKA_CFGCR0_DDI_CLK_OFF(port) |
+			 DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port));
+		I915_WRITE(DPCLKA_CFGCR0, val);
+	} else if (IS_GEN9_BC(dev_priv)) {
 		/* DDI -> PLL mapping  */
 		val = I915_READ(DPLL_CTRL2);
 
@@ -1696,6 +2056,8 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder,
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_ddi_vswing_sequence(dev_priv, level, port,
 					INTEL_OUTPUT_HDMI);
+	else if (IS_CANNONLAKE(dev_priv))
+		cnl_ddi_vswing_sequence(encoder, level);
 
 	intel_hdmi->set_infoframes(drm_encoder,
 				   has_hdmi_sink,
@@ -1732,12 +2094,18 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	enum port port = intel_ddi_get_encoder_port(intel_encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
+	struct intel_dp *intel_dp = NULL;
 	int type = intel_encoder->type;
 	uint32_t val;
 	bool wait = false;
 
 	/* old_crtc_state and old_conn_state are NULL when called from DP_MST */
 
+	if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) {
+		intel_dp = enc_to_intel_dp(encoder);
+		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
+	}
+
 	val = I915_READ(DDI_BUF_CTL(port));
 	if (val & DDI_BUF_CTL_ENABLE) {
 		val &= ~DDI_BUF_CTL_ENABLE;
@@ -1753,9 +2121,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
 	if (wait)
 		intel_wait_ddi_buf_idle(dev_priv, port);
 
-	if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
+	if (intel_dp) {
 		intel_edp_panel_vdd_on(intel_dp);
 		intel_edp_panel_off(intel_dp);
 	}
@@ -1763,7 +2129,10 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
 	if (dig_port)
 		intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain);
 
-	if (IS_GEN9_BC(dev_priv))
+	if (IS_CANNONLAKE(dev_priv))
+		I915_WRITE(DPCLKA_CFGCR0, I915_READ(DPCLKA_CFGCR0) |
+			   DPCLKA_CFGCR0_DDI_CLK_OFF(port));
+	else if (IS_GEN9_BC(dev_priv))
 		I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) |
 					DPLL_CTRL2_DDI_CLK_OFF(port)));
 	else if (INTEL_GEN(dev_priv) < 9)
@@ -1841,7 +2210,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
 		if (port == PORT_A && INTEL_GEN(dev_priv) < 9)
 			intel_dp_stop_link_train(intel_dp);
 
-		intel_edp_backlight_on(intel_dp);
+		intel_edp_backlight_on(pipe_config, conn_state);
 		intel_psr_enable(intel_dp);
 		intel_edp_drrs_enable(intel_dp, pipe_config);
 	}
@@ -1871,7 +2240,7 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder,
 
 		intel_edp_drrs_disable(intel_dp, old_crtc_state);
 		intel_psr_disable(intel_dp);
-		intel_edp_backlight_off(intel_dp);
+		intel_edp_backlight_off(old_conn_state);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 7d01dfe7faac..77d3214e1a77 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -51,6 +51,8 @@ static const char * const platform_names[] = {
 	PLATFORM_NAME(BROXTON),
 	PLATFORM_NAME(KABYLAKE),
 	PLATFORM_NAME(GEMINILAKE),
+	PLATFORM_NAME(COFFEELAKE),
+	PLATFORM_NAME(CANNONLAKE),
 };
 #undef PLATFORM_NAME
 
@@ -183,16 +185,15 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 				DIV_ROUND_UP(sseu->eu_total,
 					     sseu_subslice_total(sseu)) : 0;
 	/*
-	 * SKL supports slice power gating on devices with more than
+	 * SKL+ supports slice power gating on devices with more than
 	 * one slice, and supports EU power gating on devices with
-	 * more than one EU pair per subslice. BXT supports subslice
+	 * more than one EU pair per subslice. BXT+ supports subslice
 	 * power gating on devices with more than one subslice, and
 	 * supports EU power gating on devices with more than one EU
 	 * pair per subslice.
 	*/
 	sseu->has_slice_pg =
-		(IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) &&
-		hweight8(sseu->slice_mask) > 1;
+		!IS_GEN9_LP(dev_priv) && hweight8(sseu->slice_mask) > 1;
 	sseu->has_subslice_pg =
 		IS_GEN9_LP(dev_priv) && sseu_subslice_total(sseu) > 1;
 	sseu->has_eu_pg = sseu->eu_per_subslice > 2;
@@ -327,7 +328,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 	 * we don't expose the topmost plane at all to prevent ABI breakage
 	 * down the line.
 	 */
-	if (IS_GEMINILAKE(dev_priv))
+	if (IS_GEN10(dev_priv) || IS_GEMINILAKE(dev_priv))
 		for_each_pipe(dev_priv, pipe)
 			info->num_sprites[pipe] = 3;
 	else if (IS_BROXTON(dev_priv)) {
@@ -337,7 +338,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		for_each_pipe(dev_priv, pipe)
 			info->num_sprites[pipe] = 2;
-	} else if (INTEL_GEN(dev_priv) >= 5) {
+	} else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) {
 		for_each_pipe(dev_priv, pipe)
 			info->num_sprites[pipe] = 1;
 	}
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 9106ea32b048..dec9e58545a1 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -1193,9 +1193,8 @@ void assert_pipe(struct drm_i915_private *dev_priv,
 								      pipe);
 	enum intel_display_power_domain power_domain;
 
-	/* if we need the pipe quirk it must be always on */
-	if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
-	    (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+	/* we keep both pipes enabled on 830 */
+	if (IS_I830(dev_priv))
 		state = true;
 
 	power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
@@ -1278,7 +1277,7 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
 		I915_STATE_WARN(val & SPRITE_ENABLE,
 		     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
 		     plane_name(pipe), pipe_name(pipe));
-	} else if (INTEL_GEN(dev_priv) >= 5) {
+	} else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) {
 		u32 val = I915_READ(DVSCNTR(pipe));
 		I915_STATE_WARN(val & DVS_ENABLE,
 		     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
@@ -1550,6 +1549,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	i915_reg_t reg = DPLL(crtc->pipe);
 	u32 dpll = crtc->config->dpll_hw_state.dpll;
+	int i;
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
 
@@ -1596,15 +1596,11 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
 	}
 
 	/* We do this three times for luck */
-	I915_WRITE(reg, dpll);
-	POSTING_READ(reg);
-	udelay(150); /* wait for warmup */
-	I915_WRITE(reg, dpll);
-	POSTING_READ(reg);
-	udelay(150); /* wait for warmup */
-	I915_WRITE(reg, dpll);
-	POSTING_READ(reg);
-	udelay(150); /* wait for warmup */
+	for (i = 0; i < 3; i++) {
+		I915_WRITE(reg, dpll);
+		POSTING_READ(reg);
+		udelay(150); /* wait for warmup */
+	}
 }
 
 /**
@@ -1632,8 +1628,7 @@ static void i9xx_disable_pll(struct intel_crtc *crtc)
 	}
 
 	/* Don't disable pipe or pipe PLLs if needed */
-	if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
-	    (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+	if (IS_I830(dev_priv))
 		return;
 
 	/* Make sure the pipe isn't still relying on us */
@@ -1916,8 +1911,8 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	reg = PIPECONF(cpu_transcoder);
 	val = I915_READ(reg);
 	if (val & PIPECONF_ENABLE) {
-		WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
-			  (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
+		/* we keep both pipes enabled on 830 */
+		WARN_ON(!IS_I830(dev_priv));
 		return;
 	}
 
@@ -1977,8 +1972,7 @@ static void intel_disable_pipe(struct intel_crtc *crtc)
 		val &= ~PIPECONF_DOUBLE_WIDE;
 
 	/* Don't disable pipe or pipe PLLs if needed */
-	if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
-	    !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+	if (!IS_I830(dev_priv))
 		val &= ~PIPECONF_ENABLE;
 
 	I915_WRITE(reg, val);
@@ -2085,6 +2079,18 @@ intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
 	}
 }
 
+static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
+{
+	if (IS_I830(dev_priv))
+		return 16 * 1024;
+	else if (IS_I85X(dev_priv))
+		return 256;
+	else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
+		return 32;
+	else
+		return 4 * 1024;
+}
+
 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
 {
 	if (INTEL_INFO(dev_priv)->gen >= 9)
@@ -2387,11 +2393,17 @@ u32 intel_compute_tile_offset(int *x, int *y,
 			      const struct intel_plane_state *state,
 			      int plane)
 {
-	const struct drm_i915_private *dev_priv = to_i915(state->base.plane->dev);
+	struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
+	struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
 	const struct drm_framebuffer *fb = state->base.fb;
 	unsigned int rotation = state->base.rotation;
 	int pitch = intel_fb_pitch(fb, plane, rotation);
-	u32 alignment = intel_surf_alignment(fb, plane);
+	u32 alignment;
+
+	if (intel_plane->id == PLANE_CURSOR)
+		alignment = intel_cursor_alignment(dev_priv);
+	else
+		alignment = intel_surf_alignment(fb, plane);
 
 	return _intel_compute_tile_offset(dev_priv, x, y, fb, plane, pitch,
 					  rotation, alignment);
@@ -2469,7 +2481,7 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv,
 
 		offset = _intel_compute_tile_offset(dev_priv, &x, &y,
 						    fb, i, fb->pitches[i],
-						    DRM_ROTATE_0, tile_size);
+						    DRM_MODE_ROTATE_0, tile_size);
 		offset /= tile_size;
 
 		if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
@@ -2504,7 +2516,7 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv,
 			drm_rect_rotate(&r,
 					rot_info->plane[i].width * tile_width,
 					rot_info->plane[i].height * tile_height,
-					DRM_ROTATE_270);
+					DRM_MODE_ROTATE_270);
 			x = r.x1;
 			y = r.y1;
 
@@ -2751,7 +2763,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
 				false);
 	intel_pre_disable_primary_noatomic(&intel_crtc->base);
 	trace_intel_disable_plane(primary, intel_crtc);
-	intel_plane->disable_plane(primary, &intel_crtc->base);
+	intel_plane->disable_plane(intel_plane, intel_crtc);
 
 	return;
 
@@ -2940,7 +2952,7 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state)
 	if (drm_rotation_90_or_270(rotation))
 		drm_rect_rotate(&plane_state->base.src,
 				fb->width << 16, fb->height << 16,
-				DRM_ROTATE_270);
+				DRM_MODE_ROTATE_270);
 
 	/*
 	 * Handle the AUX surface first since
@@ -2982,10 +2994,8 @@ static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
 
-	if (INTEL_GEN(dev_priv) < 4) {
-		if (crtc->pipe == PIPE_B)
-			dspcntr |= DISPPLANE_SEL_PIPE_B;
-	}
+	if (INTEL_GEN(dev_priv) < 4)
+		dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
 
 	switch (fb->format->format) {
 	case DRM_FORMAT_C8:
@@ -3018,10 +3028,10 @@ static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
 	    fb->modifier == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 
-	if (rotation & DRM_ROTATE_180)
+	if (rotation & DRM_MODE_ROTATE_180)
 		dspcntr |= DISPPLANE_ROTATE_180;
 
-	if (rotation & DRM_REFLECT_X)
+	if (rotation & DRM_MODE_REFLECT_X)
 		dspcntr |= DISPPLANE_MIRROR;
 
 	return dspcntr;
@@ -3049,10 +3059,10 @@ int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
 		int src_w = drm_rect_width(&plane_state->base.src) >> 16;
 		int src_h = drm_rect_height(&plane_state->base.src) >> 16;
 
-		if (rotation & DRM_ROTATE_180) {
+		if (rotation & DRM_MODE_ROTATE_180) {
 			src_x += src_w - 1;
 			src_y += src_h - 1;
-		} else if (rotation & DRM_REFLECT_X) {
+		} else if (rotation & DRM_MODE_REFLECT_X) {
 			src_x += src_w - 1;
 		}
 	}
@@ -3064,14 +3074,14 @@ int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
 	return 0;
 }
 
-static void i9xx_update_primary_plane(struct drm_plane *primary,
+static void i9xx_update_primary_plane(struct intel_plane *primary,
 				      const struct intel_crtc_state *crtc_state,
 				      const struct intel_plane_state *plane_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(primary->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
-	struct drm_framebuffer *fb = plane_state->base.fb;
-	int plane = intel_crtc->plane;
+	struct drm_i915_private *dev_priv = to_i915(primary->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum plane plane = primary->plane;
 	u32 linear_offset;
 	u32 dspcntr = plane_state->ctl;
 	i915_reg_t reg = DSPCNTR(plane);
@@ -3082,12 +3092,12 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
 
 	if (INTEL_GEN(dev_priv) >= 4)
-		intel_crtc->dspaddr_offset = plane_state->main.offset;
+		crtc->dspaddr_offset = plane_state->main.offset;
 	else
-		intel_crtc->dspaddr_offset = linear_offset;
+		crtc->dspaddr_offset = linear_offset;
 
-	intel_crtc->adjusted_x = x;
-	intel_crtc->adjusted_y = y;
+	crtc->adjusted_x = x;
+	crtc->adjusted_y = y;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
@@ -3113,31 +3123,29 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		I915_WRITE_FW(DSPSURF(plane),
 			      intel_plane_ggtt_offset(plane_state) +
-			      intel_crtc->dspaddr_offset);
+			      crtc->dspaddr_offset);
 		I915_WRITE_FW(DSPOFFSET(plane), (y << 16) | x);
 	} else if (INTEL_GEN(dev_priv) >= 4) {
 		I915_WRITE_FW(DSPSURF(plane),
 			      intel_plane_ggtt_offset(plane_state) +
-			      intel_crtc->dspaddr_offset);
+			      crtc->dspaddr_offset);
 		I915_WRITE_FW(DSPTILEOFF(plane), (y << 16) | x);
 		I915_WRITE_FW(DSPLINOFF(plane), linear_offset);
 	} else {
 		I915_WRITE_FW(DSPADDR(plane),
 			      intel_plane_ggtt_offset(plane_state) +
-			      intel_crtc->dspaddr_offset);
+			      crtc->dspaddr_offset);
 	}
 	POSTING_READ_FW(reg);
 
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static void i9xx_disable_primary_plane(struct drm_plane *primary,
-				       struct drm_crtc *crtc)
+static void i9xx_disable_primary_plane(struct intel_plane *primary,
+				       struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int plane = intel_crtc->plane;
+	struct drm_i915_private *dev_priv = to_i915(primary->base.dev);
+	enum plane plane = primary->plane;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
@@ -3272,17 +3280,17 @@ static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
 static u32 skl_plane_ctl_rotation(unsigned int rotation)
 {
 	switch (rotation) {
-	case DRM_ROTATE_0:
+	case DRM_MODE_ROTATE_0:
 		break;
 	/*
-	 * DRM_ROTATE_ is counter clockwise to stay compatible with Xrandr
+	 * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
 	 * while i915 HW rotation is clockwise, thats why this swapping.
 	 */
-	case DRM_ROTATE_90:
+	case DRM_MODE_ROTATE_90:
 		return PLANE_CTL_ROTATE_270;
-	case DRM_ROTATE_180:
+	case DRM_MODE_ROTATE_180:
 		return PLANE_CTL_ROTATE_180;
-	case DRM_ROTATE_270:
+	case DRM_MODE_ROTATE_270:
 		return PLANE_CTL_ROTATE_90;
 	default:
 		MISSING_CASE(rotation);
@@ -3322,16 +3330,15 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 	return plane_ctl;
 }
 
-static void skylake_update_primary_plane(struct drm_plane *plane,
+static void skylake_update_primary_plane(struct intel_plane *plane,
 					 const struct intel_crtc_state *crtc_state,
 					 const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = plane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
-	struct drm_framebuffer *fb = plane_state->base.fb;
-	enum plane_id plane_id = to_intel_plane(plane)->id;
-	enum pipe pipe = to_intel_plane(plane)->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum plane_id plane_id = plane->id;
+	enum pipe pipe = plane->pipe;
 	u32 plane_ctl = plane_state->ctl;
 	unsigned int rotation = plane_state->base.rotation;
 	u32 stride = skl_plane_stride(fb, 0, rotation);
@@ -3353,10 +3360,10 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 	dst_w--;
 	dst_h--;
 
-	intel_crtc->dspaddr_offset = surf_addr;
+	crtc->dspaddr_offset = surf_addr;
 
-	intel_crtc->adjusted_x = src_x;
-	intel_crtc->adjusted_y = src_y;
+	crtc->adjusted_x = src_x;
+	crtc->adjusted_y = src_y;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
@@ -3395,13 +3402,12 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static void skylake_disable_primary_plane(struct drm_plane *primary,
-					  struct drm_crtc *crtc)
+static void skylake_disable_primary_plane(struct intel_plane *primary,
+					  struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum plane_id plane_id = to_intel_plane(primary)->id;
-	enum pipe pipe = to_intel_plane(primary)->pipe;
+	struct drm_i915_private *dev_priv = to_i915(primary->base.dev);
+	enum plane_id plane_id = primary->id;
+	enum pipe pipe = primary->pipe;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
@@ -3434,7 +3440,7 @@ static void intel_update_primary_planes(struct drm_device *dev)
 			trace_intel_update_plane(&plane->base,
 						 to_intel_crtc(crtc));
 
-			plane->update_plane(&plane->base,
+			plane->update_plane(plane,
 					    to_intel_crtc_state(crtc->state),
 					    plane_state);
 		}
@@ -4864,12 +4870,9 @@ static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
 {
 	if (intel_crtc->overlay) {
 		struct drm_device *dev = intel_crtc->base.dev;
-		struct drm_i915_private *dev_priv = to_i915(dev);
 
 		mutex_lock(&dev->struct_mutex);
-		dev_priv->mm.interruptible = false;
 		(void) intel_overlay_switch_off(intel_crtc->overlay);
-		dev_priv->mm.interruptible = true;
 		mutex_unlock(&dev->struct_mutex);
 	}
 
@@ -5089,7 +5092,7 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask
 	intel_crtc_dpms_overlay_disable(intel_crtc);
 
 	drm_for_each_plane_mask(p, dev, plane_mask)
-		to_intel_plane(p)->disable_plane(p, crtc);
+		to_intel_plane(p)->disable_plane(to_intel_plane(p), intel_crtc);
 
 	/*
 	 * FIXME: Once we grow proper nuclear flip support out of this we need
@@ -5725,6 +5728,8 @@ static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 			     struct drm_atomic_state *old_state)
 {
+	struct intel_atomic_state *old_intel_state =
+		to_intel_atomic_state(old_state);
 	struct drm_crtc *crtc = pipe_config->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -5757,7 +5762,11 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_color_load_luts(&pipe_config->base);
 
-	intel_update_watermarks(intel_crtc);
+	if (dev_priv->display.initial_watermarks != NULL)
+		dev_priv->display.initial_watermarks(old_intel_state,
+						     intel_crtc->config);
+	else
+		intel_update_watermarks(intel_crtc);
 	intel_enable_pipe(intel_crtc);
 
 	assert_vblank_disabled(crtc);
@@ -5824,6 +5833,10 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
 
 	if (!dev_priv->display.initial_watermarks)
 		intel_update_watermarks(intel_crtc);
+
+	/* clock the pipe down to 640x480@60 to potentially save power */
+	if (IS_I830(dev_priv))
+		i830_enable_pipe(dev_priv, pipe);
 }
 
 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
@@ -5924,9 +5937,10 @@ void intel_encoder_destroy(struct drm_encoder *encoder)
 
 /* Cross check the actual hw state with our own modeset state tracking (and it's
  * internal consistency). */
-static void intel_connector_verify_state(struct intel_connector *connector)
+static void intel_connector_verify_state(struct drm_crtc_state *crtc_state,
+					 struct drm_connector_state *conn_state)
 {
-	struct drm_crtc *crtc = connector->base.state->crtc;
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
 		      connector->base.base.id,
@@ -5934,15 +5948,14 @@ static void intel_connector_verify_state(struct intel_connector *connector)
 
 	if (connector->get_hw_state(connector)) {
 		struct intel_encoder *encoder = connector->encoder;
-		struct drm_connector_state *conn_state = connector->base.state;
 
-		I915_STATE_WARN(!crtc,
+		I915_STATE_WARN(!crtc_state,
 			 "connector enabled without attached crtc\n");
 
-		if (!crtc)
+		if (!crtc_state)
 			return;
 
-		I915_STATE_WARN(!crtc->state->active,
+		I915_STATE_WARN(!crtc_state->active,
 		      "connector is active, but attached crtc isn't\n");
 
 		if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
@@ -5954,20 +5967,30 @@ static void intel_connector_verify_state(struct intel_connector *connector)
 		I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
 			"attached encoder crtc differs from connector crtc\n");
 	} else {
-		I915_STATE_WARN(crtc && crtc->state->active,
+		I915_STATE_WARN(crtc_state && crtc_state->active,
 			"attached crtc is active, but connector isn't\n");
-		I915_STATE_WARN(!crtc && connector->base.state->best_encoder,
+		I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
 			"best encoder set without crtc!\n");
 	}
 }
 
 int intel_connector_init(struct intel_connector *connector)
 {
-	drm_atomic_helper_connector_reset(&connector->base);
+	struct intel_digital_connector_state *conn_state;
 
-	if (!connector->base.state)
+	/*
+	 * Allocate enough memory to hold intel_digital_connector_state,
+	 * This might be a few bytes too many, but for connectors that don't
+	 * need it we'll free the state and allocate a smaller one on the first
+	 * succesful commit anyway.
+	 */
+	conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL);
+	if (!conn_state)
 		return -ENOMEM;
 
+	__drm_atomic_helper_connector_reset(&connector->base,
+					    &conn_state->base);
+
 	return 0;
 }
 
@@ -6382,8 +6405,8 @@ static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
 	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
 
 	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
-	reg_val &= 0x8cffffff;
-	reg_val = 0x8c000000;
+	reg_val &= 0x00ffffff;
+	reg_val |= 0x8c000000;
 	vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
 
 	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
@@ -7025,8 +7048,8 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 
 	pipeconf = 0;
 
-	if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
-	    (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+	/* we keep both pipes enabled on 830 */
+	if (IS_I830(dev_priv))
 		pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
 
 	if (intel_crtc->config->double_wide)
@@ -8187,9 +8210,6 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct dpll reduced_clock;
-	bool has_reduced_clock = false;
-	struct intel_shared_dpll *pll;
 	const struct intel_limit *limit;
 	int refclk = 120000;
 
@@ -8231,20 +8251,14 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
 		return -EINVAL;
 	}
 
-	ironlake_compute_dpll(crtc, crtc_state,
-			      has_reduced_clock ? &reduced_clock : NULL);
+	ironlake_compute_dpll(crtc, crtc_state, NULL);
 
-	pll = intel_get_shared_dpll(crtc, crtc_state, NULL);
-	if (pll == NULL) {
+	if (!intel_get_shared_dpll(crtc, crtc_state, NULL)) {
 		DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n",
 				 pipe_name(crtc->pipe));
 		return -EINVAL;
 	}
 
-	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
-	    has_reduced_clock)
-		crtc->lowfreq_avail = true;
-
 	return 0;
 }
 
@@ -8860,6 +8874,22 @@ static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
 	return 0;
 }
 
+static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
+				   enum port port,
+				   struct intel_crtc_state *pipe_config)
+{
+	enum intel_dpll_id id;
+	u32 temp;
+
+	temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
+	id = temp >> (port * 2);
+
+	if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
+		return;
+
+	pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
+}
+
 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
 				enum port port,
 				struct intel_crtc_state *pipe_config)
@@ -9047,7 +9077,9 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
 	port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
 
-	if (IS_GEN9_BC(dev_priv))
+	if (IS_CANNONLAKE(dev_priv))
+		cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
+	else if (IS_GEN9_BC(dev_priv))
 		skylake_get_ddi_pll(dev_priv, port, pipe_config);
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_get_ddi_pll(dev_priv, port, pipe_config);
@@ -9148,38 +9180,171 @@ out:
 	return active;
 }
 
+static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
+{
+	struct drm_i915_private *dev_priv =
+		to_i915(plane_state->base.plane->dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+	u32 base;
+
+	if (INTEL_INFO(dev_priv)->cursor_needs_physical)
+		base = obj->phys_handle->busaddr;
+	else
+		base = intel_plane_ggtt_offset(plane_state);
+
+	base += plane_state->main.offset;
+
+	/* ILK+ do this automagically */
+	if (HAS_GMCH_DISPLAY(dev_priv) &&
+	    plane_state->base.rotation & DRM_MODE_ROTATE_180)
+		base += (plane_state->base.crtc_h *
+			 plane_state->base.crtc_w - 1) * fb->format->cpp[0];
+
+	return base;
+}
+
+static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
+{
+	int x = plane_state->base.crtc_x;
+	int y = plane_state->base.crtc_y;
+	u32 pos = 0;
+
+	if (x < 0) {
+		pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
+		x = -x;
+	}
+	pos |= x << CURSOR_X_SHIFT;
+
+	if (y < 0) {
+		pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
+		y = -y;
+	}
+	pos |= y << CURSOR_Y_SHIFT;
+
+	return pos;
+}
+
+static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
+{
+	const struct drm_mode_config *config =
+		&plane_state->base.plane->dev->mode_config;
+	int width = plane_state->base.crtc_w;
+	int height = plane_state->base.crtc_h;
+
+	return width > 0 && width <= config->cursor_width &&
+		height > 0 && height <= config->cursor_height;
+}
+
+static int intel_check_cursor(struct intel_crtc_state *crtc_state,
+			      struct intel_plane_state *plane_state)
+{
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	int src_x, src_y;
+	u32 offset;
+	int ret;
+
+	ret = drm_plane_helper_check_state(&plane_state->base,
+					   &plane_state->clip,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   true, true);
+	if (ret)
+		return ret;
+
+	if (!fb)
+		return 0;
+
+	if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
+		DRM_DEBUG_KMS("cursor cannot be tiled\n");
+		return -EINVAL;
+	}
+
+	src_x = plane_state->base.src_x >> 16;
+	src_y = plane_state->base.src_y >> 16;
+
+	intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
+	offset = intel_compute_tile_offset(&src_x, &src_y, plane_state, 0);
+
+	if (src_x != 0 || src_y != 0) {
+		DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
+		return -EINVAL;
+	}
+
+	plane_state->main.offset = offset;
+
+	return 0;
+}
+
 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
 			   const struct intel_plane_state *plane_state)
 {
-	unsigned int width = plane_state->base.crtc_w;
-	unsigned int stride = roundup_pow_of_two(width) * 4;
+	const struct drm_framebuffer *fb = plane_state->base.fb;
 
-	switch (stride) {
-	default:
-		WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
-			  width, stride);
-		stride = 256;
-		/* fallthrough */
+	return CURSOR_ENABLE |
+		CURSOR_GAMMA_ENABLE |
+		CURSOR_FORMAT_ARGB |
+		CURSOR_STRIDE(fb->pitches[0]);
+}
+
+static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
+{
+	int width = plane_state->base.crtc_w;
+
+	/*
+	 * 845g/865g are only limited by the width of their cursors,
+	 * the height is arbitrary up to the precision of the register.
+	 */
+	return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
+}
+
+static int i845_check_cursor(struct intel_plane *plane,
+			     struct intel_crtc_state *crtc_state,
+			     struct intel_plane_state *plane_state)
+{
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	int ret;
+
+	ret = intel_check_cursor(crtc_state, plane_state);
+	if (ret)
+		return ret;
+
+	/* if we want to turn off the cursor ignore width and height */
+	if (!fb)
+		return 0;
+
+	/* Check for which cursor types we support */
+	if (!i845_cursor_size_ok(plane_state)) {
+		DRM_DEBUG("Cursor dimension %dx%d not supported\n",
+			  plane_state->base.crtc_w,
+			  plane_state->base.crtc_h);
+		return -EINVAL;
+	}
+
+	switch (fb->pitches[0]) {
 	case 256:
 	case 512:
 	case 1024:
 	case 2048:
 		break;
+	default:
+		DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
+			      fb->pitches[0]);
+		return -EINVAL;
 	}
 
-	return CURSOR_ENABLE |
-		CURSOR_GAMMA_ENABLE |
-		CURSOR_FORMAT_ARGB |
-		CURSOR_STRIDE(stride);
+	plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
+
+	return 0;
 }
 
-static void i845_update_cursor(struct drm_crtc *crtc, u32 base,
+static void i845_update_cursor(struct intel_plane *plane,
+			       const struct intel_crtc_state *crtc_state,
 			       const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	uint32_t cntl = 0, size = 0;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	u32 cntl = 0, base = 0, pos = 0, size = 0;
+	unsigned long irqflags;
 
 	if (plane_state && plane_state->base.visible) {
 		unsigned int width = plane_state->base.crtc_w;
@@ -9187,35 +9352,41 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base,
 
 		cntl = plane_state->ctl;
 		size = (height << 12) | width;
-	}
 
-	if (intel_crtc->cursor_cntl != 0 &&
-	    (intel_crtc->cursor_base != base ||
-	     intel_crtc->cursor_size != size ||
-	     intel_crtc->cursor_cntl != cntl)) {
-		/* On these chipsets we can only modify the base/size/stride
-		 * whilst the cursor is disabled.
-		 */
-		I915_WRITE_FW(CURCNTR(PIPE_A), 0);
-		POSTING_READ_FW(CURCNTR(PIPE_A));
-		intel_crtc->cursor_cntl = 0;
+		base = intel_cursor_base(plane_state);
+		pos = intel_cursor_position(plane_state);
 	}
 
-	if (intel_crtc->cursor_base != base) {
-		I915_WRITE_FW(CURBASE(PIPE_A), base);
-		intel_crtc->cursor_base = base;
-	}
+	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-	if (intel_crtc->cursor_size != size) {
+	/* On these chipsets we can only modify the base/size/stride
+	 * whilst the cursor is disabled.
+	 */
+	if (plane->cursor.base != base ||
+	    plane->cursor.size != size ||
+	    plane->cursor.cntl != cntl) {
+		I915_WRITE_FW(CURCNTR(PIPE_A), 0);
+		I915_WRITE_FW(CURBASE(PIPE_A), base);
 		I915_WRITE_FW(CURSIZE, size);
-		intel_crtc->cursor_size = size;
-	}
-
-	if (intel_crtc->cursor_cntl != cntl) {
+		I915_WRITE_FW(CURPOS(PIPE_A), pos);
 		I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
-		POSTING_READ_FW(CURCNTR(PIPE_A));
-		intel_crtc->cursor_cntl = cntl;
+
+		plane->cursor.base = base;
+		plane->cursor.size = size;
+		plane->cursor.cntl = cntl;
+	} else {
+		I915_WRITE_FW(CURPOS(PIPE_A), pos);
 	}
+
+	POSTING_READ_FW(CURCNTR(PIPE_A));
+
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+}
+
+static void i845_disable_cursor(struct intel_plane *plane,
+				struct intel_crtc *crtc)
+{
+	i845_update_cursor(plane, NULL, NULL);
 }
 
 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
@@ -9224,7 +9395,6 @@ static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv =
 		to_i915(plane_state->base.plane->dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
-	enum pipe pipe = crtc->pipe;
 	u32 cntl;
 
 	cntl = MCURSOR_GAMMA_ENABLE;
@@ -9232,7 +9402,7 @@ static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
 	if (HAS_DDI(dev_priv))
 		cntl |= CURSOR_PIPE_CSC_ENABLE;
 
-	cntl |= pipe << 28; /* Connect to correct pipe */
+	cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
 
 	switch (plane_state->base.crtc_w) {
 	case 64:
@@ -9249,122 +9419,160 @@ static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
 		return 0;
 	}
 
-	if (plane_state->base.rotation & DRM_ROTATE_180)
+	if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
 		cntl |= CURSOR_ROTATE_180;
 
 	return cntl;
 }
 
-static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
-			       const struct intel_plane_state *plane_state)
+static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int pipe = intel_crtc->pipe;
-	uint32_t cntl = 0;
+	struct drm_i915_private *dev_priv =
+		to_i915(plane_state->base.plane->dev);
+	int width = plane_state->base.crtc_w;
+	int height = plane_state->base.crtc_h;
 
-	if (plane_state && plane_state->base.visible)
-		cntl = plane_state->ctl;
+	if (!intel_cursor_size_ok(plane_state))
+		return false;
 
-	if (intel_crtc->cursor_cntl != cntl) {
-		I915_WRITE_FW(CURCNTR(pipe), cntl);
-		POSTING_READ_FW(CURCNTR(pipe));
-		intel_crtc->cursor_cntl = cntl;
+	/* Cursor width is limited to a few power-of-two sizes */
+	switch (width) {
+	case 256:
+	case 128:
+	case 64:
+		break;
+	default:
+		return false;
 	}
 
-	/* and commit changes on next vblank */
-	I915_WRITE_FW(CURBASE(pipe), base);
-	POSTING_READ_FW(CURBASE(pipe));
+	/*
+	 * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
+	 * height from 8 lines up to the cursor width, when the
+	 * cursor is not rotated. Everything else requires square
+	 * cursors.
+	 */
+	if (HAS_CUR_FBC(dev_priv) &&
+	    plane_state->base.rotation & DRM_MODE_ROTATE_0) {
+		if (height < 8 || height > width)
+			return false;
+	} else {
+		if (height != width)
+			return false;
+	}
 
-	intel_crtc->cursor_base = base;
+	return true;
 }
 
-/* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
-static void intel_crtc_update_cursor(struct drm_crtc *crtc,
-				     const struct intel_plane_state *plane_state)
+static int i9xx_check_cursor(struct intel_plane *plane,
+			     struct intel_crtc_state *crtc_state,
+			     struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int pipe = intel_crtc->pipe;
-	u32 base = intel_crtc->cursor_addr;
-	unsigned long irqflags;
-	u32 pos = 0;
-
-	if (plane_state) {
-		int x = plane_state->base.crtc_x;
-		int y = plane_state->base.crtc_y;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum pipe pipe = plane->pipe;
+	int ret;
 
-		if (x < 0) {
-			pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
-			x = -x;
-		}
-		pos |= x << CURSOR_X_SHIFT;
+	ret = intel_check_cursor(crtc_state, plane_state);
+	if (ret)
+		return ret;
 
-		if (y < 0) {
-			pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
-			y = -y;
-		}
-		pos |= y << CURSOR_Y_SHIFT;
+	/* if we want to turn off the cursor ignore width and height */
+	if (!fb)
+		return 0;
 
-		/* ILK+ do this automagically */
-		if (HAS_GMCH_DISPLAY(dev_priv) &&
-		    plane_state->base.rotation & DRM_ROTATE_180) {
-			base += (plane_state->base.crtc_h *
-				 plane_state->base.crtc_w - 1) * 4;
-		}
+	/* Check for which cursor types we support */
+	if (!i9xx_cursor_size_ok(plane_state)) {
+		DRM_DEBUG("Cursor dimension %dx%d not supported\n",
+			  plane_state->base.crtc_w,
+			  plane_state->base.crtc_h);
+		return -EINVAL;
 	}
 
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+	if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
+		DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
+			      fb->pitches[0], plane_state->base.crtc_w);
+		return -EINVAL;
+	}
 
-	I915_WRITE_FW(CURPOS(pipe), pos);
+	/*
+	 * There's something wrong with the cursor on CHV pipe C.
+	 * If it straddles the left edge of the screen then
+	 * moving it away from the edge or disabling it often
+	 * results in a pipe underrun, and often that can lead to
+	 * dead pipe (constant underrun reported, and it scans
+	 * out just a solid color). To recover from that, the
+	 * display power well must be turned off and on again.
+	 * Refuse the put the cursor into that compromised position.
+	 */
+	if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
+	    plane_state->base.visible && plane_state->base.crtc_x < 0) {
+		DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
+		return -EINVAL;
+	}
 
-	if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
-		i845_update_cursor(crtc, base, plane_state);
-	else
-		i9xx_update_cursor(crtc, base, plane_state);
+	plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
 
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+	return 0;
 }
 
-static bool cursor_size_ok(struct drm_i915_private *dev_priv,
-			   uint32_t width, uint32_t height)
+static void i9xx_update_cursor(struct intel_plane *plane,
+			       const struct intel_crtc_state *crtc_state,
+			       const struct intel_plane_state *plane_state)
 {
-	if (width == 0 || height == 0)
-		return false;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum pipe pipe = plane->pipe;
+	u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
+	unsigned long irqflags;
 
-	/*
-	 * 845g/865g are special in that they are only limited by
-	 * the width of their cursors, the height is arbitrary up to
-	 * the precision of the register. Everything else requires
-	 * square cursors, limited to a few power-of-two sizes.
-	 */
-	if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
-		if ((width & 63) != 0)
-			return false;
+	if (plane_state && plane_state->base.visible) {
+		cntl = plane_state->ctl;
 
-		if (width > (IS_I845G(dev_priv) ? 64 : 512))
-			return false;
+		if (plane_state->base.crtc_h != plane_state->base.crtc_w)
+			fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
 
-		if (height > 1023)
-			return false;
+		base = intel_cursor_base(plane_state);
+		pos = intel_cursor_position(plane_state);
+	}
+
+	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+
+	/*
+	 * On some platforms writing CURCNTR first will also
+	 * cause CURPOS to be armed by the CURBASE write.
+	 * Without the CURCNTR write the CURPOS write would
+	 * arm itself.
+	 *
+	 * CURCNTR and CUR_FBC_CTL are always
+	 * armed by the CURBASE write only.
+	 */
+	if (plane->cursor.base != base ||
+	    plane->cursor.size != fbc_ctl ||
+	    plane->cursor.cntl != cntl) {
+		I915_WRITE_FW(CURCNTR(pipe), cntl);
+		if (HAS_CUR_FBC(dev_priv))
+			I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
+		I915_WRITE_FW(CURPOS(pipe), pos);
+		I915_WRITE_FW(CURBASE(pipe), base);
+
+		plane->cursor.base = base;
+		plane->cursor.size = fbc_ctl;
+		plane->cursor.cntl = cntl;
 	} else {
-		switch (width | height) {
-		case 256:
-		case 128:
-			if (IS_GEN2(dev_priv))
-				return false;
-		case 64:
-			break;
-		default:
-			return false;
-		}
+		I915_WRITE_FW(CURPOS(pipe), pos);
 	}
 
-	return true;
+	POSTING_READ_FW(CURBASE(pipe));
+
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
+static void i9xx_disable_cursor(struct intel_plane *plane,
+				struct intel_crtc *crtc)
+{
+	i9xx_update_cursor(plane, NULL, NULL);
+}
+
+
 /* VESA 640x480x72Hz mode to set on the pipe */
 static struct drm_display_mode load_detect_mode = {
 	DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
@@ -9576,6 +9784,7 @@ int intel_get_load_detect_pipe(struct drm_connector *connector,
 	 */
 	if (!crtc) {
 		DRM_DEBUG_KMS("no pipe available for load-detect\n");
+		ret = -ENODEV;
 		goto fail;
 	}
 
@@ -9632,6 +9841,7 @@ found:
 		DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
 	if (IS_ERR(fb)) {
 		DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
+		ret = PTR_ERR(fb);
 		goto fail;
 	}
 
@@ -10863,21 +11073,21 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 			 turn_off, turn_on, mode_changed);
 
 	if (turn_on) {
-		if (INTEL_GEN(dev_priv) < 5)
+		if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
 			pipe_config->update_wm_pre = true;
 
 		/* must disable cxsr around plane enable/disable */
 		if (plane->id != PLANE_CURSOR)
 			pipe_config->disable_cxsr = true;
 	} else if (turn_off) {
-		if (INTEL_GEN(dev_priv) < 5)
+		if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
 			pipe_config->update_wm_post = true;
 
 		/* must disable cxsr around plane enable/disable */
 		if (plane->id != PLANE_CURSOR)
 			pipe_config->disable_cxsr = true;
 	} else if (intel_wm_need_update(&plane->base, plane_state)) {
-		if (INTEL_GEN(dev_priv) < 5) {
+		if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
 			/* FIXME bollocks */
 			pipe_config->update_wm_pre = true;
 			pipe_config->update_wm_post = true;
@@ -11003,6 +11213,9 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 			ret = skl_update_scaler_crtc(pipe_config);
 
 		if (!ret)
+			ret = skl_check_pipe_max_pixel_rate(intel_crtc,
+							    pipe_config);
+		if (!ret)
 			ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
 							 pipe_config);
 	}
@@ -11226,6 +11439,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
 	unsigned int used_ports = 0;
 	unsigned int used_mst_ports = 0;
 
@@ -11234,7 +11448,8 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 	 * list to detect the problem on ddi platforms
 	 * where there's just one encoder per digital port.
 	 */
-	drm_for_each_connector(connector, dev) {
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
 		struct drm_connector_state *connector_state;
 		struct intel_encoder *encoder;
 
@@ -11273,6 +11488,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 			break;
 		}
 	}
+	drm_connector_list_iter_end(&conn_iter);
 
 	/* can't mix MST and SST/HDMI on the same port */
 	if (used_ports & used_mst_ports)
@@ -11301,7 +11517,8 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
 	shared_dpll = crtc_state->shared_dpll;
 	dpll_hw_state = crtc_state->dpll_hw_state;
 	force_thru = crtc_state->pch_pfit.force_thru;
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+	if (IS_G4X(dev_priv) ||
+	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		wm_state = crtc_state->wm;
 
 	/* Keep base drm_crtc_state intact, only clear our extended struct */
@@ -11313,7 +11530,8 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
 	crtc_state->shared_dpll = shared_dpll;
 	crtc_state->dpll_hw_state = dpll_hw_state;
 	crtc_state->pch_pfit.force_thru = force_thru;
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+	if (IS_G4X(dev_priv) ||
+	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		crtc_state->wm = wm_state;
 }
 
@@ -11454,12 +11672,6 @@ intel_modeset_update_crtc_state(struct drm_atomic_state *state)
 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
 		to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
 
-		/* Update hwmode for vblank functions */
-		if (new_crtc_state->active)
-			crtc->hwmode = new_crtc_state->adjusted_mode;
-		else
-			crtc->hwmode.crtc_clock = 0;
-
 		/*
 		 * Update legacy state to satisfy fbc code. This can
 		 * be removed when fbc uses the atomic state.
@@ -11881,7 +12093,7 @@ static void verify_wm_state(struct drm_crtc *crtc,
 	 * allocation. In that case since the ddb allocation will be updated
 	 * once the plane becomes visible, we can skip this check
 	 */
-	if (intel_crtc->cursor_addr) {
+	if (1) {
 		hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
 		sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
 
@@ -11937,11 +12149,15 @@ verify_connector_state(struct drm_device *dev,
 
 	for_each_new_connector_in_state(state, connector, new_conn_state, i) {
 		struct drm_encoder *encoder = connector->encoder;
+		struct drm_crtc_state *crtc_state = NULL;
 
 		if (new_conn_state->crtc != crtc)
 			continue;
 
-		intel_connector_verify_state(to_intel_connector(connector));
+		if (crtc)
+			crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
+
+		intel_connector_verify_state(crtc_state, new_conn_state);
 
 		I915_STATE_WARN(new_conn_state->best_encoder != encoder,
 		     "connector's atomic encoder doesn't match legacy encoder\n");
@@ -12021,9 +12237,8 @@ verify_crtc_state(struct drm_crtc *crtc,
 
 	active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
 
-	/* hw state is inconsistent with the pipe quirk */
-	if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
-	    (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+	/* we keep both pipes enabled on 830 */
+	if (IS_I830(dev_priv))
 		active = new_crtc_state->active;
 
 	I915_STATE_WARN(new_crtc_state->active != active,
@@ -12059,7 +12274,7 @@ verify_crtc_state(struct drm_crtc *crtc,
 
 	intel_pipe_config_sanity_check(dev_priv, pipe_config);
 
-	sw_config = to_intel_crtc_state(crtc->state);
+	sw_config = to_intel_crtc_state(new_crtc_state);
 	if (!intel_pipe_config_compare(dev_priv, sw_config,
 				       pipe_config, false)) {
 		I915_STATE_WARN(1, "pipe state doesn't match!\n");
@@ -12932,8 +13147,16 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_hw_done(state);
 
-	if (intel_state->modeset)
+	if (intel_state->modeset) {
+		/* As one of the primary mmio accessors, KMS has a high
+		 * likelihood of triggering bugs in unclaimed access. After we
+		 * finish modesetting, see if an error has been flagged, and if
+		 * so enable debugging for the next modeset - and hope we catch
+		 * the culprit.
+		 */
+		intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
 		intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
+	}
 
 	mutex_lock(&dev->struct_mutex);
 	drm_atomic_helper_cleanup_planes(dev, state);
@@ -12943,19 +13166,6 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_state_put(state);
 
-	/* As one of the primary mmio accessors, KMS has a high likelihood
-	 * of triggering bugs in unclaimed access. After we finish
-	 * modesetting, see if an error has been flagged, and if so
-	 * enable debugging for the next modeset - and hope we catch
-	 * the culprit.
-	 *
-	 * XXX note that we assume display power is on at this point.
-	 * This might hold true now but we need to add pm helper to check
-	 * unclaimed only when the hardware is on, as atomic commits
-	 * can happen also when the device is completely off.
-	 */
-	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
-
 	intel_atomic_helper_free_state(dev_priv);
 }
 
@@ -13087,43 +13297,6 @@ static int intel_atomic_commit(struct drm_device *dev,
 	return 0;
 }
 
-void intel_crtc_restore_mode(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_atomic_state *state;
-	struct drm_crtc_state *crtc_state;
-	int ret;
-
-	state = drm_atomic_state_alloc(dev);
-	if (!state) {
-		DRM_DEBUG_KMS("[CRTC:%d:%s] crtc restore failed, out of memory",
-			      crtc->base.id, crtc->name);
-		return;
-	}
-
-	state->acquire_ctx = crtc->dev->mode_config.acquire_ctx;
-
-retry:
-	crtc_state = drm_atomic_get_crtc_state(state, crtc);
-	ret = PTR_ERR_OR_ZERO(crtc_state);
-	if (!ret) {
-		if (!crtc_state->active)
-			goto out;
-
-		crtc_state->mode_changed = true;
-		ret = drm_atomic_commit(state);
-	}
-
-	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
-		drm_modeset_backoff(state->acquire_ctx);
-		goto retry;
-	}
-
-out:
-	drm_atomic_state_put(state);
-}
-
 static const struct drm_crtc_funcs intel_crtc_funcs = {
 	.gamma_set = drm_atomic_helper_legacy_gamma_set,
 	.set_config = drm_atomic_helper_set_config,
@@ -13164,7 +13337,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 	if (obj) {
 		if (plane->type == DRM_PLANE_TYPE_CURSOR &&
 		    INTEL_INFO(dev_priv)->cursor_needs_physical) {
-			const int align = IS_I830(dev_priv) ? 16 * 1024 : 256;
+			const int align = intel_cursor_alignment(dev_priv);
 
 			ret = i915_gem_object_attach_phys(obj, align);
 			if (ret) {
@@ -13294,11 +13467,11 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state
 }
 
 static int
-intel_check_primary_plane(struct drm_plane *plane,
+intel_check_primary_plane(struct intel_plane *plane,
 			  struct intel_crtc_state *crtc_state,
 			  struct intel_plane_state *state)
 {
-	struct drm_i915_private *dev_priv = to_i915(plane->dev);
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	struct drm_crtc *crtc = state->base.crtc;
 	int min_scale = DRM_PLANE_HELPER_NO_SCALING;
 	int max_scale = DRM_PLANE_HELPER_NO_SCALING;
@@ -13477,7 +13650,7 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 		goto out_free;
 
 	if (INTEL_INFO(dev_priv)->cursor_needs_physical) {
-		int align = IS_I830(dev_priv) ? 16 * 1024 : 256;
+		int align = intel_cursor_alignment(dev_priv);
 
 		ret = i915_gem_object_attach_phys(intel_fb_obj(fb), align);
 		if (ret) {
@@ -13513,12 +13686,12 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 
 	if (plane->state->visible) {
 		trace_intel_update_plane(plane, to_intel_crtc(crtc));
-		intel_plane->update_plane(plane,
+		intel_plane->update_plane(intel_plane,
 					  to_intel_crtc_state(crtc->state),
 					  to_intel_plane_state(plane->state));
 	} else {
 		trace_intel_disable_plane(plane, to_intel_crtc(crtc));
-		intel_plane->disable_plane(plane, crtc);
+		intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc));
 	}
 
 	intel_cleanup_plane_fb(plane, new_plane_state);
@@ -13632,22 +13805,22 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 
 	if (INTEL_GEN(dev_priv) >= 9) {
 		supported_rotations =
-			DRM_ROTATE_0 | DRM_ROTATE_90 |
-			DRM_ROTATE_180 | DRM_ROTATE_270;
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+			DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270;
 	} else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
 		supported_rotations =
-			DRM_ROTATE_0 | DRM_ROTATE_180 |
-			DRM_REFLECT_X;
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
+			DRM_MODE_REFLECT_X;
 	} else if (INTEL_GEN(dev_priv) >= 4) {
 		supported_rotations =
-			DRM_ROTATE_0 | DRM_ROTATE_180;
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
 	} else {
-		supported_rotations = DRM_ROTATE_0;
+		supported_rotations = DRM_MODE_ROTATE_0;
 	}
 
 	if (INTEL_GEN(dev_priv) >= 4)
 		drm_plane_create_rotation_property(&primary->base,
-						   DRM_ROTATE_0,
+						   DRM_MODE_ROTATE_0,
 						   supported_rotations);
 
 	drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
@@ -13661,107 +13834,9 @@ fail:
 	return ERR_PTR(ret);
 }
 
-static int
-intel_check_cursor_plane(struct drm_plane *plane,
-			 struct intel_crtc_state *crtc_state,
-			 struct intel_plane_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(plane->dev);
-	struct drm_framebuffer *fb = state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
-	enum pipe pipe = to_intel_plane(plane)->pipe;
-	unsigned stride;
-	int ret;
-
-	ret = drm_plane_helper_check_state(&state->base,
-					   &state->clip,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   true, true);
-	if (ret)
-		return ret;
-
-	/* if we want to turn off the cursor ignore width and height */
-	if (!obj)
-		return 0;
-
-	/* Check for which cursor types we support */
-	if (!cursor_size_ok(dev_priv, state->base.crtc_w,
-			    state->base.crtc_h)) {
-		DRM_DEBUG("Cursor dimension %dx%d not supported\n",
-			  state->base.crtc_w, state->base.crtc_h);
-		return -EINVAL;
-	}
-
-	stride = roundup_pow_of_two(state->base.crtc_w) * 4;
-	if (obj->base.size < stride * state->base.crtc_h) {
-		DRM_DEBUG_KMS("buffer is too small\n");
-		return -ENOMEM;
-	}
-
-	if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
-		DRM_DEBUG_KMS("cursor cannot be tiled\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * There's something wrong with the cursor on CHV pipe C.
-	 * If it straddles the left edge of the screen then
-	 * moving it away from the edge or disabling it often
-	 * results in a pipe underrun, and often that can lead to
-	 * dead pipe (constant underrun reported, and it scans
-	 * out just a solid color). To recover from that, the
-	 * display power well must be turned off and on again.
-	 * Refuse the put the cursor into that compromised position.
-	 */
-	if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
-	    state->base.visible && state->base.crtc_x < 0) {
-		DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
-		return -EINVAL;
-	}
-
-	if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
-		state->ctl = i845_cursor_ctl(crtc_state, state);
-	else
-		state->ctl = i9xx_cursor_ctl(crtc_state, state);
-
-	return 0;
-}
-
-static void
-intel_disable_cursor_plane(struct drm_plane *plane,
-			   struct drm_crtc *crtc)
-{
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
-	intel_crtc->cursor_addr = 0;
-	intel_crtc_update_cursor(crtc, NULL);
-}
-
-static void
-intel_update_cursor_plane(struct drm_plane *plane,
-			  const struct intel_crtc_state *crtc_state,
-			  const struct intel_plane_state *state)
-{
-	struct drm_crtc *crtc = crtc_state->base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct drm_i915_private *dev_priv = to_i915(plane->dev);
-	struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
-	uint32_t addr;
-
-	if (!obj)
-		addr = 0;
-	else if (!INTEL_INFO(dev_priv)->cursor_needs_physical)
-		addr = intel_plane_ggtt_offset(state);
-	else
-		addr = obj->phys_handle->busaddr;
-
-	intel_crtc->cursor_addr = addr;
-	intel_crtc_update_cursor(crtc, state);
-}
-
 static struct intel_plane *
-intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
+intel_cursor_plane_create(struct drm_i915_private *dev_priv,
+			  enum pipe pipe)
 {
 	struct intel_plane *cursor = NULL;
 	struct intel_plane_state *state = NULL;
@@ -13787,9 +13862,22 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 	cursor->plane = pipe;
 	cursor->id = PLANE_CURSOR;
 	cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe);
-	cursor->check_plane = intel_check_cursor_plane;
-	cursor->update_plane = intel_update_cursor_plane;
-	cursor->disable_plane = intel_disable_cursor_plane;
+
+	if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
+		cursor->update_plane = i845_update_cursor;
+		cursor->disable_plane = i845_disable_cursor;
+		cursor->check_plane = i845_check_cursor;
+	} else {
+		cursor->update_plane = i9xx_update_cursor;
+		cursor->disable_plane = i9xx_disable_cursor;
+		cursor->check_plane = i9xx_check_cursor;
+	}
+
+	cursor->cursor.base = ~0;
+	cursor->cursor.cntl = ~0;
+
+	if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
+		cursor->cursor.size = ~0;
 
 	ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
 				       0, &intel_cursor_plane_funcs,
@@ -13802,9 +13890,9 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 
 	if (INTEL_GEN(dev_priv) >= 4)
 		drm_plane_create_rotation_property(&cursor->base,
-						   DRM_ROTATE_0,
-						   DRM_ROTATE_0 |
-						   DRM_ROTATE_180);
+						   DRM_MODE_ROTATE_0,
+						   DRM_MODE_ROTATE_0 |
+						   DRM_MODE_ROTATE_180);
 
 	if (INTEL_GEN(dev_priv) >= 9)
 		state->scaler_id = -1;
@@ -13898,10 +13986,6 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 	intel_crtc->pipe = pipe;
 	intel_crtc->plane = primary->plane;
 
-	intel_crtc->cursor_base = ~0;
-	intel_crtc->cursor_cntl = ~0;
-	intel_crtc->cursor_size = ~0;
-
 	/* initialize shared scalers */
 	intel_crtc_init_scalers(intel_crtc, crtc_state);
 
@@ -14441,7 +14525,7 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_VYUY:
-		if (INTEL_GEN(dev_priv) < 5) {
+		if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
 			DRM_DEBUG_KMS("unsupported pixel format: %s\n",
 				      drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			goto err;
@@ -14653,27 +14737,6 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
 }
 
 /*
- * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
- * resume, or other times.  This quirk makes sure that's the case for
- * affected systems.
- */
-static void quirk_pipea_force(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	dev_priv->quirks |= QUIRK_PIPEA_FORCE;
-	DRM_INFO("applying pipe a force quirk\n");
-}
-
-static void quirk_pipeb_force(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	dev_priv->quirks |= QUIRK_PIPEB_FORCE;
-	DRM_INFO("applying pipe b force quirk\n");
-}
-
-/*
  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
  */
 static void quirk_ssc_force_disable(struct drm_device *dev)
@@ -14738,18 +14801,6 @@ static const struct intel_dmi_quirk intel_dmi_quirks[] = {
 };
 
 static struct intel_quirk intel_quirks[] = {
-	/* Toshiba Protege R-205, S-209 needs pipe A force quirk */
-	{ 0x2592, 0x1179, 0x0001, quirk_pipea_force },
-
-	/* ThinkPad T60 needs pipe A force quirk (bug #16494) */
-	{ 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
-
-	/* 830 needs to leave pipe A & dpll A up */
-	{ 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
-
-	/* 830 needs to leave pipe B & dpll B up */
-	{ 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
-
 	/* Lenovo U160 cannot use SSC on LVDS */
 	{ 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
 
@@ -14953,6 +15004,7 @@ int intel_modeset_init(struct drm_device *dev)
 
 	dev->mode_config.funcs = &intel_mode_funcs;
 
+	init_llist_head(&dev_priv->atomic_helper.free_list);
 	INIT_WORK(&dev_priv->atomic_helper.free_work,
 		  intel_atomic_helper_free_state_worker);
 
@@ -15069,35 +15121,89 @@ int intel_modeset_init(struct drm_device *dev)
 	return 0;
 }
 
-static void intel_enable_pipe_a(struct drm_device *dev,
-				struct drm_modeset_acquire_ctx *ctx)
+void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	struct intel_connector *connector;
-	struct drm_connector_list_iter conn_iter;
-	struct drm_connector *crt = NULL;
-	struct intel_load_detect_pipe load_detect_temp;
-	int ret;
+	/* 640x480@60Hz, ~25175 kHz */
+	struct dpll clock = {
+		.m1 = 18,
+		.m2 = 7,
+		.p1 = 13,
+		.p2 = 4,
+		.n = 2,
+	};
+	u32 dpll, fp;
+	int i;
 
-	/* We can't just switch on the pipe A, we need to set things up with a
-	 * proper mode and output configuration. As a gross hack, enable pipe A
-	 * by enabling the load detect pipe once. */
-	drm_connector_list_iter_begin(dev, &conn_iter);
-	for_each_intel_connector_iter(connector, &conn_iter) {
-		if (connector->encoder->type == INTEL_OUTPUT_ANALOG) {
-			crt = &connector->base;
-			break;
-		}
+	WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
+
+	DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
+		      pipe_name(pipe), clock.vco, clock.dot);
+
+	fp = i9xx_dpll_compute_fp(&clock);
+	dpll = (I915_READ(DPLL(pipe)) & DPLL_DVO_2X_MODE) |
+		DPLL_VGA_MODE_DIS |
+		((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
+		PLL_P2_DIVIDE_BY_4 |
+		PLL_REF_INPUT_DREFCLK |
+		DPLL_VCO_ENABLE;
+
+	I915_WRITE(FP0(pipe), fp);
+	I915_WRITE(FP1(pipe), fp);
+
+	I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
+	I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
+	I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
+	I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
+	I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
+	I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
+	I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
+
+	/*
+	 * Apparently we need to have VGA mode enabled prior to changing
+	 * the P1/P2 dividers. Otherwise the DPLL will keep using the old
+	 * dividers, even though the register value does change.
+	 */
+	I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
+	I915_WRITE(DPLL(pipe), dpll);
+
+	/* Wait for the clocks to stabilize. */
+	POSTING_READ(DPLL(pipe));
+	udelay(150);
+
+	/* The pixel multiplier can only be updated once the
+	 * DPLL is enabled and the clocks are stable.
+	 *
+	 * So write it again.
+	 */
+	I915_WRITE(DPLL(pipe), dpll);
+
+	/* We do this three times for luck */
+	for (i = 0; i < 3 ; i++) {
+		I915_WRITE(DPLL(pipe), dpll);
+		POSTING_READ(DPLL(pipe));
+		udelay(150); /* wait for warmup */
 	}
-	drm_connector_list_iter_end(&conn_iter);
 
-	if (!crt)
-		return;
+	I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
+	POSTING_READ(PIPECONF(pipe));
+}
+
+void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
+{
+	DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
+		      pipe_name(pipe));
 
-	ret = intel_get_load_detect_pipe(crt, NULL, &load_detect_temp, ctx);
-	WARN(ret < 0, "All modeset mutexes are locked, but intel_get_load_detect_pipe failed\n");
+	assert_plane_disabled(dev_priv, PLANE_A);
+	assert_plane_disabled(dev_priv, PLANE_B);
 
-	if (ret > 0)
-		intel_release_load_detect_pipe(crt, &load_detect_temp, ctx);
+	I915_WRITE(PIPECONF(pipe), 0);
+	POSTING_READ(PIPECONF(pipe));
+
+	if (wait_for(pipe_dsl_stopped(dev_priv, pipe), 100))
+		DRM_ERROR("pipe %c off wait timed out\n", pipe_name(pipe));
+
+	I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
+	POSTING_READ(DPLL(pipe));
 }
 
 static bool
@@ -15175,7 +15281,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc,
 				continue;
 
 			trace_intel_disable_plane(&plane->base, crtc);
-			plane->disable_plane(&plane->base, &crtc->base);
+			plane->disable_plane(plane, crtc);
 		}
 	}
 
@@ -15198,15 +15304,6 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc,
 		crtc->plane = plane;
 	}
 
-	if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
-	    crtc->pipe == PIPE_A && !crtc->active) {
-		/* BIOS forgot to enable pipe A, this mostly happens after
-		 * resume. Force-enable the pipe to fix this, the update_dpms
-		 * call below we restore the pipe to the right state, but leave
-		 * the required bits on. */
-		intel_enable_pipe_a(dev, ctx);
-	}
-
 	/* Adjust the state of the output pipe according to whether we
 	 * have active connectors/encoders. */
 	if (crtc->active && !intel_crtc_has_encoders(crtc))
@@ -15445,8 +15542,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 			to_intel_crtc_state(crtc->base.state);
 		int pixclk = 0;
 
-		crtc->base.hwmode = crtc_state->base.adjusted_mode;
-
 		memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
 		if (crtc_state->base.active) {
 			intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
@@ -15476,7 +15571,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 			if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled)
 				pixclk = DIV_ROUND_UP(pixclk * 100, 95);
 
-			drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode);
+			drm_calc_timestamping_constants(&crtc->base,
+							&crtc_state->base.adjusted_mode);
 			update_scanline_offset(crtc);
 		}
 
@@ -15548,7 +15644,10 @@ intel_modeset_setup_hw_state(struct drm_device *dev,
 		pll->on = false;
 	}
 
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+	if (IS_G4X(dev_priv)) {
+		g4x_wm_get_hw_state(dev);
+		g4x_wm_sanitize(dev_priv);
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		vlv_wm_get_hw_state(dev);
 		vlv_wm_sanitize(dev_priv);
 	} else if (IS_GEN9(dev_priv)) {
@@ -15582,13 +15681,6 @@ void intel_display_resume(struct drm_device *dev)
 	if (state)
 		state->acquire_ctx = &ctx;
 
-	/*
-	 * This is a cludge because with real atomic modeset mode_config.mutex
-	 * won't be taken. Unfortunately some probed state like
-	 * audio_codec_enable is still protected by mode_config.mutex, so lock
-	 * it here for now.
-	 */
-	mutex_lock(&dev->mode_config.mutex);
 	drm_modeset_acquire_init(&ctx, 0);
 
 	while (1) {
@@ -15604,7 +15696,6 @@ void intel_display_resume(struct drm_device *dev)
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
-	mutex_unlock(&dev->mode_config.mutex);
 
 	if (ret)
 		DRM_ERROR("Restoring old state failed with %i\n", ret);
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index fc691b8b317c..64fa774c855b 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -133,36 +133,55 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
 				      enum pipe pipe);
 static void intel_dp_unset_edid(struct intel_dp *intel_dp);
 
-static int
-intel_dp_max_link_bw(struct intel_dp  *intel_dp)
+static int intel_dp_num_rates(u8 link_bw_code)
 {
-	int max_link_bw = intel_dp->dpcd[DP_MAX_LINK_RATE];
-
-	switch (max_link_bw) {
+	switch (link_bw_code) {
+	default:
+		WARN(1, "invalid max DP link bw val %x, using 1.62Gbps\n",
+		     link_bw_code);
 	case DP_LINK_BW_1_62:
+		return 1;
 	case DP_LINK_BW_2_7:
+		return 2;
 	case DP_LINK_BW_5_4:
-		break;
-	default:
-		WARN(1, "invalid max DP link bw val %x, using 1.62Gbps\n",
-		     max_link_bw);
-		max_link_bw = DP_LINK_BW_1_62;
-		break;
+		return 3;
 	}
-	return max_link_bw;
 }
 
-static u8 intel_dp_max_lane_count(struct intel_dp *intel_dp)
+/* update sink rates from dpcd */
+static void intel_dp_set_sink_rates(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	u8 source_max, sink_max;
+	int i, num_rates;
+
+	num_rates = intel_dp_num_rates(intel_dp->dpcd[DP_MAX_LINK_RATE]);
 
-	source_max = intel_dig_port->max_lanes;
-	sink_max = intel_dp->max_sink_lane_count;
+	for (i = 0; i < num_rates; i++)
+		intel_dp->sink_rates[i] = default_rates[i];
+
+	intel_dp->num_sink_rates = num_rates;
+}
+
+/* Theoretical max between source and sink */
+static int intel_dp_max_common_rate(struct intel_dp *intel_dp)
+{
+	return intel_dp->common_rates[intel_dp->num_common_rates - 1];
+}
+
+/* Theoretical max between source and sink */
+static int intel_dp_max_common_lane_count(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+	int source_max = intel_dig_port->max_lanes;
+	int sink_max = drm_dp_max_lane_count(intel_dp->dpcd);
 
 	return min(source_max, sink_max);
 }
 
+int intel_dp_max_lane_count(struct intel_dp *intel_dp)
+{
+	return intel_dp->max_link_lane_count;
+}
+
 int
 intel_dp_link_required(int pixel_clock, int bpp)
 {
@@ -205,34 +224,25 @@ intel_dp_downstream_max_dotclock(struct intel_dp *intel_dp)
 	return max_dotclk;
 }
 
-static int
-intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates)
-{
-	if (intel_dp->num_sink_rates) {
-		*sink_rates = intel_dp->sink_rates;
-		return intel_dp->num_sink_rates;
-	}
-
-	*sink_rates = default_rates;
-
-	return (intel_dp->max_sink_link_bw >> 3) + 1;
-}
-
-static int
-intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates)
+static void
+intel_dp_set_source_rates(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
+	const int *source_rates;
 	int size;
 
+	/* This should only be done once */
+	WARN_ON(intel_dp->source_rates || intel_dp->num_source_rates);
+
 	if (IS_GEN9_LP(dev_priv)) {
-		*source_rates = bxt_rates;
+		source_rates = bxt_rates;
 		size = ARRAY_SIZE(bxt_rates);
 	} else if (IS_GEN9_BC(dev_priv)) {
-		*source_rates = skl_rates;
+		source_rates = skl_rates;
 		size = ARRAY_SIZE(skl_rates);
 	} else {
-		*source_rates = default_rates;
+		source_rates = default_rates;
 		size = ARRAY_SIZE(default_rates);
 	}
 
@@ -240,7 +250,8 @@ intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates)
 	if (!intel_dp_source_supports_hbr2(intel_dp))
 		size--;
 
-	return size;
+	intel_dp->source_rates = source_rates;
+	intel_dp->num_source_rates = size;
 }
 
 static int intersect_rates(const int *source_rates, int source_len,
@@ -266,50 +277,83 @@ static int intersect_rates(const int *source_rates, int source_len,
 	return k;
 }
 
-static int intel_dp_common_rates(struct intel_dp *intel_dp,
-				 int *common_rates)
+/* return index of rate in rates array, or -1 if not found */
+static int intel_dp_rate_index(const int *rates, int len, int rate)
 {
-	const int *source_rates, *sink_rates;
-	int source_len, sink_len;
+	int i;
 
-	sink_len = intel_dp_sink_rates(intel_dp, &sink_rates);
-	source_len = intel_dp_source_rates(intel_dp, &source_rates);
+	for (i = 0; i < len; i++)
+		if (rate == rates[i])
+			return i;
 
-	return intersect_rates(source_rates, source_len,
-			       sink_rates, sink_len,
-			       common_rates);
+	return -1;
 }
 
-static int intel_dp_link_rate_index(struct intel_dp *intel_dp,
-				    int *common_rates, int link_rate)
+static void intel_dp_set_common_rates(struct intel_dp *intel_dp)
 {
-	int common_len;
-	int index;
+	WARN_ON(!intel_dp->num_source_rates || !intel_dp->num_sink_rates);
 
-	common_len = intel_dp_common_rates(intel_dp, common_rates);
-	for (index = 0; index < common_len; index++) {
-		if (link_rate == common_rates[common_len - index - 1])
-			return common_len - index - 1;
+	intel_dp->num_common_rates = intersect_rates(intel_dp->source_rates,
+						     intel_dp->num_source_rates,
+						     intel_dp->sink_rates,
+						     intel_dp->num_sink_rates,
+						     intel_dp->common_rates);
+
+	/* Paranoia, there should always be something in common. */
+	if (WARN_ON(intel_dp->num_common_rates == 0)) {
+		intel_dp->common_rates[0] = default_rates[0];
+		intel_dp->num_common_rates = 1;
 	}
+}
 
-	return -1;
+/* get length of common rates potentially limited by max_rate */
+static int intel_dp_common_len_rate_limit(struct intel_dp *intel_dp,
+					  int max_rate)
+{
+	const int *common_rates = intel_dp->common_rates;
+	int i, common_len = intel_dp->num_common_rates;
+
+	/* Limit results by potentially reduced max rate */
+	for (i = 0; i < common_len; i++) {
+		if (common_rates[common_len - i - 1] <= max_rate)
+			return common_len - i;
+	}
+
+	return 0;
+}
+
+static bool intel_dp_link_params_valid(struct intel_dp *intel_dp)
+{
+	/*
+	 * FIXME: we need to synchronize the current link parameters with
+	 * hardware readout. Currently fast link training doesn't work on
+	 * boot-up.
+	 */
+	if (intel_dp->link_rate == 0 ||
+	    intel_dp->link_rate > intel_dp->max_link_rate)
+		return false;
+
+	if (intel_dp->lane_count == 0 ||
+	    intel_dp->lane_count > intel_dp_max_lane_count(intel_dp))
+		return false;
+
+	return true;
 }
 
 int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
 					    int link_rate, uint8_t lane_count)
 {
-	int common_rates[DP_MAX_SUPPORTED_RATES];
-	int link_rate_index;
+	int index;
 
-	link_rate_index = intel_dp_link_rate_index(intel_dp,
-						   common_rates,
-						   link_rate);
-	if (link_rate_index > 0) {
-		intel_dp->max_sink_link_bw = drm_dp_link_rate_to_bw_code(common_rates[link_rate_index - 1]);
-		intel_dp->max_sink_lane_count = lane_count;
+	index = intel_dp_rate_index(intel_dp->common_rates,
+				    intel_dp->num_common_rates,
+				    link_rate);
+	if (index > 0) {
+		intel_dp->max_link_rate = intel_dp->common_rates[index - 1];
+		intel_dp->max_link_lane_count = lane_count;
 	} else if (lane_count > 1) {
-		intel_dp->max_sink_link_bw = intel_dp_max_link_bw(intel_dp);
-		intel_dp->max_sink_lane_count = lane_count >> 1;
+		intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp);
+		intel_dp->max_link_lane_count = lane_count >> 1;
 	} else {
 		DRM_ERROR("Link Training Unsuccessful\n");
 		return -1;
@@ -754,7 +798,7 @@ static void intel_pps_get_registers(struct drm_i915_private *dev_priv,
 	regs->pp_stat = PP_STATUS(pps_idx);
 	regs->pp_on = PP_ON_DELAYS(pps_idx);
 	regs->pp_off = PP_OFF_DELAYS(pps_idx);
-	if (!IS_GEN9_LP(dev_priv))
+	if (!IS_GEN9_LP(dev_priv) && !HAS_PCH_CNP(dev_priv))
 		regs->pp_div = PP_DIVISOR(pps_idx);
 }
 
@@ -1486,60 +1530,52 @@ static void snprintf_int_array(char *str, size_t len,
 
 static void intel_dp_print_rates(struct intel_dp *intel_dp)
 {
-	const int *source_rates, *sink_rates;
-	int source_len, sink_len, common_len;
-	int common_rates[DP_MAX_SUPPORTED_RATES];
 	char str[128]; /* FIXME: too big for stack? */
 
 	if ((drm_debug & DRM_UT_KMS) == 0)
 		return;
 
-	source_len = intel_dp_source_rates(intel_dp, &source_rates);
-	snprintf_int_array(str, sizeof(str), source_rates, source_len);
+	snprintf_int_array(str, sizeof(str),
+			   intel_dp->source_rates, intel_dp->num_source_rates);
 	DRM_DEBUG_KMS("source rates: %s\n", str);
 
-	sink_len = intel_dp_sink_rates(intel_dp, &sink_rates);
-	snprintf_int_array(str, sizeof(str), sink_rates, sink_len);
+	snprintf_int_array(str, sizeof(str),
+			   intel_dp->sink_rates, intel_dp->num_sink_rates);
 	DRM_DEBUG_KMS("sink rates: %s\n", str);
 
-	common_len = intel_dp_common_rates(intel_dp, common_rates);
-	snprintf_int_array(str, sizeof(str), common_rates, common_len);
+	snprintf_int_array(str, sizeof(str),
+			   intel_dp->common_rates, intel_dp->num_common_rates);
 	DRM_DEBUG_KMS("common rates: %s\n", str);
 }
 
-static int rate_to_index(int find, const int *rates)
-{
-	int i = 0;
-
-	for (i = 0; i < DP_MAX_SUPPORTED_RATES; ++i)
-		if (find == rates[i])
-			break;
-
-	return i;
-}
-
 int
 intel_dp_max_link_rate(struct intel_dp *intel_dp)
 {
-	int rates[DP_MAX_SUPPORTED_RATES] = {};
 	int len;
 
-	len = intel_dp_common_rates(intel_dp, rates);
+	len = intel_dp_common_len_rate_limit(intel_dp, intel_dp->max_link_rate);
 	if (WARN_ON(len <= 0))
 		return 162000;
 
-	return rates[len - 1];
+	return intel_dp->common_rates[len - 1];
 }
 
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate)
 {
-	return rate_to_index(rate, intel_dp->sink_rates);
+	int i = intel_dp_rate_index(intel_dp->sink_rates,
+				    intel_dp->num_sink_rates, rate);
+
+	if (WARN_ON(i < 0))
+		i = 0;
+
+	return i;
 }
 
 void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
 			   uint8_t *link_bw, uint8_t *rate_select)
 {
-	if (intel_dp->num_sink_rates) {
+	/* eDP 1.4 rate select method. */
+	if (intel_dp->use_rate_select) {
 		*link_bw = 0;
 		*rate_select =
 			intel_dp_rate_select(intel_dp, port_clock);
@@ -1581,22 +1617,23 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	enum port port = dp_to_dig_port(intel_dp)->port;
 	struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
 	struct intel_connector *intel_connector = intel_dp->attached_connector;
+	struct intel_digital_connector_state *intel_conn_state =
+		to_intel_digital_connector_state(conn_state);
 	int lane_count, clock;
 	int min_lane_count = 1;
 	int max_lane_count = intel_dp_max_lane_count(intel_dp);
 	/* Conveniently, the link BW constants become indices with a shift...*/
 	int min_clock = 0;
 	int max_clock;
-	int link_rate_index;
 	int bpp, mode_rate;
 	int link_avail, link_clock;
-	int common_rates[DP_MAX_SUPPORTED_RATES] = {};
 	int common_len;
 	uint8_t link_bw, rate_select;
 	bool reduce_m_n = drm_dp_has_quirk(&intel_dp->desc,
 					   DP_DPCD_QUIRK_LIMITED_M_N);
 
-	common_len = intel_dp_common_rates(intel_dp, common_rates);
+	common_len = intel_dp_common_len_rate_limit(intel_dp,
+						    intel_dp->max_link_rate);
 
 	/* No common link rates between source and sink */
 	WARN_ON(common_len <= 0);
@@ -1607,7 +1644,12 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		pipe_config->has_pch_encoder = true;
 
 	pipe_config->has_drrs = false;
-	pipe_config->has_audio = intel_dp->has_audio && port != PORT_A;
+	if (port == PORT_A)
+		pipe_config->has_audio = false;
+	else if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO)
+		pipe_config->has_audio = intel_dp->has_audio;
+	else
+		pipe_config->has_audio = intel_conn_state->force_audio == HDMI_AUDIO_ON;
 
 	if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
 		intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
@@ -1622,10 +1664,10 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 		if (HAS_GMCH_DISPLAY(dev_priv))
 			intel_gmch_panel_fitting(intel_crtc, pipe_config,
-						 intel_connector->panel.fitting_mode);
+						 conn_state->scaling_mode);
 		else
 			intel_pch_panel_fitting(intel_crtc, pipe_config,
-						intel_connector->panel.fitting_mode);
+						conn_state->scaling_mode);
 	}
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK)
@@ -1633,16 +1675,18 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 	/* Use values requested by Compliance Test Request */
 	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
-		link_rate_index = intel_dp_link_rate_index(intel_dp,
-							   common_rates,
-							   intel_dp->compliance.test_link_rate);
-		if (link_rate_index >= 0)
-			min_clock = max_clock = link_rate_index;
+		int index;
+
+		index = intel_dp_rate_index(intel_dp->common_rates,
+					    intel_dp->num_common_rates,
+					    intel_dp->compliance.test_link_rate);
+		if (index >= 0)
+			min_clock = max_clock = index;
 		min_lane_count = max_lane_count = intel_dp->compliance.test_lane_count;
 	}
 	DRM_DEBUG_KMS("DP link computation with max lane count %i "
 		      "max bw %d pixel clock %iKHz\n",
-		      max_lane_count, common_rates[max_clock],
+		      max_lane_count, intel_dp->common_rates[max_clock],
 		      adjusted_mode->crtc_clock);
 
 	/* Walk through all bpp values. Luckily they're all nicely spaced with 2
@@ -1678,7 +1722,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 				lane_count <= max_lane_count;
 				lane_count <<= 1) {
 
-				link_clock = common_rates[clock];
+				link_clock = intel_dp->common_rates[clock];
 				link_avail = intel_dp_max_data_rate(link_clock,
 								    lane_count);
 
@@ -1692,7 +1736,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	return false;
 
 found:
-	if (intel_dp->color_range_auto) {
+	if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) {
 		/*
 		 * See:
 		 * CEA-861-E - 5.1 Default Encoding Parameters
@@ -1704,13 +1748,13 @@ found:
 			HDMI_QUANTIZATION_RANGE_LIMITED;
 	} else {
 		pipe_config->limited_color_range =
-			intel_dp->limited_color_range;
+			intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_LIMITED;
 	}
 
 	pipe_config->lane_count = lane_count;
 
 	pipe_config->pipe_bpp = bpp;
-	pipe_config->port_clock = common_rates[clock];
+	pipe_config->port_clock = intel_dp->common_rates[clock];
 
 	intel_dp_compute_rate(intel_dp, pipe_config->port_clock,
 			      &link_bw, &rate_select);
@@ -2267,14 +2311,17 @@ static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
 }
 
 /* Enable backlight PWM and backlight PP control. */
-void intel_edp_backlight_on(struct intel_dp *intel_dp)
+void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
+			    const struct drm_connector_state *conn_state)
 {
+	struct intel_dp *intel_dp = enc_to_intel_dp(conn_state->best_encoder);
+
 	if (!is_edp(intel_dp))
 		return;
 
 	DRM_DEBUG_KMS("\n");
 
-	intel_panel_enable_backlight(intel_dp->attached_connector);
+	intel_panel_enable_backlight(crtc_state, conn_state);
 	_intel_edp_backlight_on(intel_dp);
 }
 
@@ -2306,15 +2353,17 @@ static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
 }
 
 /* Disable backlight PP control and backlight PWM. */
-void intel_edp_backlight_off(struct intel_dp *intel_dp)
+void intel_edp_backlight_off(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_dp *intel_dp = enc_to_intel_dp(old_conn_state->best_encoder);
+
 	if (!is_edp(intel_dp))
 		return;
 
 	DRM_DEBUG_KMS("\n");
 
 	_intel_edp_backlight_off(intel_dp);
-	intel_panel_disable_backlight(intel_dp->attached_connector);
+	intel_panel_disable_backlight(old_conn_state);
 }
 
 /*
@@ -2610,7 +2659,7 @@ static void intel_disable_dp(struct intel_encoder *encoder,
 	/* Make sure the panel is off before trying to change the mode. But also
 	 * ensure that we have vdd while we switch off the panel. */
 	intel_edp_panel_vdd_on(intel_dp);
-	intel_edp_backlight_off(intel_dp);
+	intel_edp_backlight_off(old_conn_state);
 	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
 	intel_edp_panel_off(intel_dp);
 
@@ -2824,10 +2873,8 @@ static void g4x_enable_dp(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config,
 			  struct drm_connector_state *conn_state)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
 	intel_enable_dp(encoder, pipe_config, conn_state);
-	intel_edp_backlight_on(intel_dp);
+	intel_edp_backlight_on(pipe_config, conn_state);
 }
 
 static void vlv_enable_dp(struct intel_encoder *encoder,
@@ -2836,7 +2883,7 @@ static void vlv_enable_dp(struct intel_encoder *encoder,
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
-	intel_edp_backlight_on(intel_dp);
+	intel_edp_backlight_on(pipe_config, conn_state);
 	intel_psr_enable(intel_dp);
 }
 
@@ -3024,7 +3071,8 @@ static bool intel_dp_get_y_cord_status(struct intel_dp *intel_dp)
 {
 	uint8_t psr_caps = 0;
 
-	drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_CAPS, &psr_caps);
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_CAPS, &psr_caps) != 1)
+		return false;
 	return psr_caps & DP_PSR2_SU_Y_COORDINATE_REQUIRED;
 }
 
@@ -3032,9 +3080,9 @@ static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp)
 {
 	uint8_t dprx = 0;
 
-	drm_dp_dpcd_readb(&intel_dp->aux,
-			DP_DPRX_FEATURE_ENUMERATION_LIST,
-			&dprx);
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST,
+			      &dprx) != 1)
+		return false;
 	return dprx & DP_VSC_SDP_EXT_FOR_COLORIMETRY_SUPPORTED;
 }
 
@@ -3042,7 +3090,9 @@ static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
 {
 	uint8_t alpm_caps = 0;
 
-	drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP, &alpm_caps);
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
+			      &alpm_caps) != 1)
+		return false;
 	return alpm_caps & DP_ALPM_CAP;
 }
 
@@ -3415,7 +3465,7 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp)
 	if (HAS_DDI(dev_priv)) {
 		signal_levels = ddi_signal_levels(intel_dp);
 
-		if (IS_GEN9_LP(dev_priv))
+		if (IS_GEN9_LP(dev_priv) || IS_CANNONLAKE(dev_priv))
 			signal_levels = 0;
 		else
 			mask = DDI_BUF_EMP_MASK;
@@ -3616,9 +3666,10 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
 		uint8_t frame_sync_cap;
 
 		dev_priv->psr.sink_support = true;
-		drm_dp_dpcd_read(&intel_dp->aux,
-				 DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
-				 &frame_sync_cap, 1);
+		if (drm_dp_dpcd_readb(&intel_dp->aux,
+				      DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
+				      &frame_sync_cap) != 1)
+			frame_sync_cap = 0;
 		dev_priv->psr.aux_frame_sync = frame_sync_cap ? true : false;
 		/* PSR2 needs frame sync as well */
 		dev_priv->psr.psr2_support = dev_priv->psr.aux_frame_sync;
@@ -3669,6 +3720,13 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
 		intel_dp->num_sink_rates = i;
 	}
 
+	if (intel_dp->num_sink_rates)
+		intel_dp->use_rate_select = true;
+	else
+		intel_dp_set_sink_rates(intel_dp);
+
+	intel_dp_set_common_rates(intel_dp);
+
 	return true;
 }
 
@@ -3676,11 +3734,18 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
 static bool
 intel_dp_get_dpcd(struct intel_dp *intel_dp)
 {
+	u8 sink_count;
+
 	if (!intel_dp_read_dpcd(intel_dp))
 		return false;
 
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT,
-			     &intel_dp->sink_count, 1) < 0)
+	/* Don't clobber cached eDP rates. */
+	if (!is_edp(intel_dp)) {
+		intel_dp_set_sink_rates(intel_dp);
+		intel_dp_set_common_rates(intel_dp);
+	}
+
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_COUNT, &sink_count) <= 0)
 		return false;
 
 	/*
@@ -3688,7 +3753,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 	 * a member variable in intel_dp will track any changes
 	 * between short pulse interrupts.
 	 */
-	intel_dp->sink_count = DP_GET_SINK_COUNT(intel_dp->sink_count);
+	intel_dp->sink_count = DP_GET_SINK_COUNT(sink_count);
 
 	/*
 	 * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that
@@ -3717,7 +3782,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 static bool
 intel_dp_can_mst(struct intel_dp *intel_dp)
 {
-	u8 buf[1];
+	u8 mstm_cap;
 
 	if (!i915.enable_dp_mst)
 		return false;
@@ -3728,10 +3793,10 @@ intel_dp_can_mst(struct intel_dp *intel_dp)
 	if (intel_dp->dpcd[DP_DPCD_REV] < 0x12)
 		return false;
 
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_MSTM_CAP, buf, 1) != 1)
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_MSTM_CAP, &mstm_cap) != 1)
 		return false;
 
-	return buf[0] & DP_MST_CAP;
+	return mstm_cap & DP_MST_CAP;
 }
 
 static void
@@ -3877,9 +3942,8 @@ stop:
 static bool
 intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector)
 {
-	return drm_dp_dpcd_read(&intel_dp->aux,
-				       DP_DEVICE_SERVICE_IRQ_VECTOR,
-				       sink_irq_vector, 1) == 1;
+	return drm_dp_dpcd_readb(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR,
+				 sink_irq_vector) == 1;
 }
 
 static bool
@@ -3900,7 +3964,6 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 {
 	int status = 0;
 	int min_lane_count = 1;
-	int common_rates[DP_MAX_SUPPORTED_RATES] = {};
 	int link_rate_index, test_link_rate;
 	uint8_t test_lane_count, test_link_bw;
 	/* (DP CTS 1.2)
@@ -3917,7 +3980,7 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 	test_lane_count &= DP_MAX_LANE_COUNT_MASK;
 	/* Validate the requested lane count */
 	if (test_lane_count < min_lane_count ||
-	    test_lane_count > intel_dp->max_sink_lane_count)
+	    test_lane_count > intel_dp->max_link_lane_count)
 		return DP_TEST_NAK;
 
 	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LINK_RATE,
@@ -3928,9 +3991,9 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 	}
 	/* Validate the requested link rate */
 	test_link_rate = drm_dp_bw_code_to_link_rate(test_link_bw);
-	link_rate_index = intel_dp_link_rate_index(intel_dp,
-						   common_rates,
-						   test_link_rate);
+	link_rate_index = intel_dp_rate_index(intel_dp->common_rates,
+					      intel_dp->num_common_rates,
+					      test_link_rate);
 	if (link_rate_index < 0)
 		return DP_TEST_NAK;
 
@@ -3943,13 +4006,13 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 static uint8_t intel_dp_autotest_video_pattern(struct intel_dp *intel_dp)
 {
 	uint8_t test_pattern;
-	uint16_t test_misc;
+	uint8_t test_misc;
 	__be16 h_width, v_height;
 	int status = 0;
 
 	/* Read the TEST_PATTERN (DP CTS 3.1.5) */
-	status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_PATTERN,
-				  &test_pattern, 1);
+	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_PATTERN,
+				   &test_pattern);
 	if (status <= 0) {
 		DRM_DEBUG_KMS("Test pattern read failed\n");
 		return DP_TEST_NAK;
@@ -3971,8 +4034,8 @@ static uint8_t intel_dp_autotest_video_pattern(struct intel_dp *intel_dp)
 		return DP_TEST_NAK;
 	}
 
-	status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_MISC0,
-				  &test_misc, 1);
+	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_MISC0,
+				   &test_misc);
 	if (status <= 0) {
 		DRM_DEBUG_KMS("TEST MISC read failed\n");
 		return DP_TEST_NAK;
@@ -4031,10 +4094,8 @@ static uint8_t intel_dp_autotest_edid(struct intel_dp *intel_dp)
 		 */
 		block += intel_connector->detect_edid->extensions;
 
-		if (!drm_dp_dpcd_write(&intel_dp->aux,
-					DP_TEST_EDID_CHECKSUM,
-					&block->checksum,
-					1))
+		if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_EDID_CHECKSUM,
+				       block->checksum) <= 0)
 			DRM_DEBUG_KMS("Failed to write EDID checksum\n");
 
 		test_result = DP_TEST_ACK | DP_TEST_EDID_CHECKSUM_WRITE;
@@ -4198,9 +4259,11 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 	if (!to_intel_crtc(intel_encoder->base.crtc)->active)
 		return;
 
-	/* FIXME: we need to synchronize this sort of stuff with hardware
-	 * readout. Currently fast link training doesn't work on boot-up. */
-	if (!intel_dp->lane_count)
+	/*
+	 * Validate the cached values of intel_dp->link_rate and
+	 * intel_dp->lane_count before attempting to retrain.
+	 */
+	if (!intel_dp_link_params_valid(intel_dp))
 		return;
 
 	/* Retrain if Channel EQ or CR not ok */
@@ -4523,10 +4586,7 @@ intel_dp_set_edid(struct intel_dp *intel_dp)
 	edid = intel_dp_get_edid(intel_dp);
 	intel_connector->detect_edid = edid;
 
-	if (intel_dp->force_audio != HDMI_AUDIO_AUTO)
-		intel_dp->has_audio = intel_dp->force_audio == HDMI_AUDIO_ON;
-	else
-		intel_dp->has_audio = drm_detect_monitor_audio(edid);
+	intel_dp->has_audio = drm_detect_monitor_audio(edid);
 }
 
 static void
@@ -4587,11 +4647,11 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 		      yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
 
 	if (intel_dp->reset_link_params) {
-		/* Set the max lane count for sink */
-		intel_dp->max_sink_lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
+		/* Initial max link lane count */
+		intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
 
-		/* Set the max link BW for sink */
-		intel_dp->max_sink_link_bw = intel_dp_max_link_bw(intel_dp);
+		/* Initial max link rate */
+		intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp);
 
 		intel_dp->reset_link_params = false;
 	}
@@ -4735,112 +4795,6 @@ static int intel_dp_get_modes(struct drm_connector *connector)
 	return 0;
 }
 
-static bool
-intel_dp_detect_audio(struct drm_connector *connector)
-{
-	bool has_audio = false;
-	struct edid *edid;
-
-	edid = to_intel_connector(connector)->detect_edid;
-	if (edid)
-		has_audio = drm_detect_monitor_audio(edid);
-
-	return has_audio;
-}
-
-static int
-intel_dp_set_property(struct drm_connector *connector,
-		      struct drm_property *property,
-		      uint64_t val)
-{
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	struct intel_connector *intel_connector = to_intel_connector(connector);
-	struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
-	struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base);
-	int ret;
-
-	ret = drm_object_property_set_value(&connector->base, property, val);
-	if (ret)
-		return ret;
-
-	if (property == dev_priv->force_audio_property) {
-		int i = val;
-		bool has_audio;
-
-		if (i == intel_dp->force_audio)
-			return 0;
-
-		intel_dp->force_audio = i;
-
-		if (i == HDMI_AUDIO_AUTO)
-			has_audio = intel_dp_detect_audio(connector);
-		else
-			has_audio = (i == HDMI_AUDIO_ON);
-
-		if (has_audio == intel_dp->has_audio)
-			return 0;
-
-		intel_dp->has_audio = has_audio;
-		goto done;
-	}
-
-	if (property == dev_priv->broadcast_rgb_property) {
-		bool old_auto = intel_dp->color_range_auto;
-		bool old_range = intel_dp->limited_color_range;
-
-		switch (val) {
-		case INTEL_BROADCAST_RGB_AUTO:
-			intel_dp->color_range_auto = true;
-			break;
-		case INTEL_BROADCAST_RGB_FULL:
-			intel_dp->color_range_auto = false;
-			intel_dp->limited_color_range = false;
-			break;
-		case INTEL_BROADCAST_RGB_LIMITED:
-			intel_dp->color_range_auto = false;
-			intel_dp->limited_color_range = true;
-			break;
-		default:
-			return -EINVAL;
-		}
-
-		if (old_auto == intel_dp->color_range_auto &&
-		    old_range == intel_dp->limited_color_range)
-			return 0;
-
-		goto done;
-	}
-
-	if (is_edp(intel_dp) &&
-	    property == connector->dev->mode_config.scaling_mode_property) {
-		if (val == DRM_MODE_SCALE_NONE) {
-			DRM_DEBUG_KMS("no scaling not supported\n");
-			return -EINVAL;
-		}
-		if (HAS_GMCH_DISPLAY(dev_priv) &&
-		    val == DRM_MODE_SCALE_CENTER) {
-			DRM_DEBUG_KMS("centering not supported\n");
-			return -EINVAL;
-		}
-
-		if (intel_connector->panel.fitting_mode == val) {
-			/* the eDP scaling property is not changed */
-			return 0;
-		}
-		intel_connector->panel.fitting_mode = val;
-
-		goto done;
-	}
-
-	return -EINVAL;
-
-done:
-	if (intel_encoder->base.crtc)
-		intel_crtc_restore_mode(intel_encoder->base.crtc);
-
-	return 0;
-}
-
 static int
 intel_dp_connector_register(struct drm_connector *connector)
 {
@@ -4999,19 +4953,21 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.force = intel_dp_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = intel_dp_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.atomic_get_property = intel_digital_connector_atomic_get_property,
+	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_dp_connector_register,
 	.early_unregister = intel_dp_connector_unregister,
 	.destroy = intel_dp_connector_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
 };
 
 static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = {
 	.detect_ctx = intel_dp_detect,
 	.get_modes = intel_dp_get_modes,
 	.mode_valid = intel_dp_mode_valid,
+	.atomic_check = intel_digital_connector_atomic_check,
 };
 
 static const struct drm_encoder_funcs intel_dp_enc_funcs = {
@@ -5102,22 +5058,25 @@ bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port)
 	return intel_bios_is_port_edp(dev_priv, port);
 }
 
-void
+static void
 intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connector)
 {
-	struct intel_connector *intel_connector = to_intel_connector(connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 
 	intel_attach_force_audio_property(connector);
 	intel_attach_broadcast_rgb_property(connector);
-	intel_dp->color_range_auto = true;
 
 	if (is_edp(intel_dp)) {
-		drm_mode_create_scaling_mode_property(connector->dev);
-		drm_object_attach_property(
-			&connector->base,
-			connector->dev->mode_config.scaling_mode_property,
-			DRM_MODE_SCALE_ASPECT);
-		intel_connector->panel.fitting_mode = DRM_MODE_SCALE_ASPECT;
+		u32 allowed_scalers;
+
+		allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN);
+		if (!HAS_GMCH_DISPLAY(dev_priv))
+			allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER);
+
+		drm_connector_attach_scaling_mode_property(connector, allowed_scalers);
+
+		connector->state->scaling_mode = DRM_MODE_SCALE_ASPECT;
+
 	}
 }
 
@@ -5143,7 +5102,7 @@ intel_pps_readout_hw_state(struct drm_i915_private *dev_priv,
 
 	pp_on = I915_READ(regs.pp_on);
 	pp_off = I915_READ(regs.pp_off);
-	if (!IS_GEN9_LP(dev_priv)) {
+	if (!IS_GEN9_LP(dev_priv) && !HAS_PCH_CNP(dev_priv)) {
 		I915_WRITE(regs.pp_ctrl, pp_ctl);
 		pp_div = I915_READ(regs.pp_div);
 	}
@@ -5161,7 +5120,7 @@ intel_pps_readout_hw_state(struct drm_i915_private *dev_priv,
 	seq->t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
 		   PANEL_POWER_DOWN_DELAY_SHIFT;
 
-	if (IS_GEN9_LP(dev_priv)) {
+	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv)) {
 		u16 tmp = (pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >>
 			BXT_POWER_CYCLE_DELAY_SHIFT;
 		if (tmp > 0)
@@ -5318,7 +5277,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 		 (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT);
 	/* Compute the divisor for the pp clock, simply match the Bspec
 	 * formula. */
-	if (IS_GEN9_LP(dev_priv)) {
+	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv)) {
 		pp_div = I915_READ(regs.pp_ctrl);
 		pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK;
 		pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000)
@@ -5344,7 +5303,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
 	I915_WRITE(regs.pp_on, pp_on);
 	I915_WRITE(regs.pp_off, pp_off);
-	if (IS_GEN9_LP(dev_priv))
+	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv))
 		I915_WRITE(regs.pp_ctrl, pp_div);
 	else
 		I915_WRITE(regs.pp_div, pp_div);
@@ -5352,7 +5311,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 	DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
 		      I915_READ(regs.pp_on),
 		      I915_READ(regs.pp_off),
-		      IS_GEN9_LP(dev_priv) ?
+		      (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv)) ?
 		      (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK) :
 		      I915_READ(regs.pp_div));
 }
@@ -5907,6 +5866,29 @@ intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
 	}
 }
 
+static void intel_dp_modeset_retry_work_fn(struct work_struct *work)
+{
+	struct intel_connector *intel_connector;
+	struct drm_connector *connector;
+
+	intel_connector = container_of(work, typeof(*intel_connector),
+				       modeset_retry_work);
+	connector = &intel_connector->base;
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id,
+		      connector->name);
+
+	/* Grab the locks before changing connector property*/
+	mutex_lock(&connector->dev->mode_config.mutex);
+	/* Set connector link status to BAD and send a Uevent to notify
+	 * userspace to do a modeset.
+	 */
+	drm_mode_connector_set_link_status_property(connector,
+						    DRM_MODE_LINK_STATUS_BAD);
+	mutex_unlock(&connector->dev->mode_config.mutex);
+	/* Send Hotplug uevent so userspace can reprobe */
+	drm_kms_helper_hotplug_event(connector->dev);
+}
+
 bool
 intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 			struct intel_connector *intel_connector)
@@ -5919,11 +5901,17 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	enum port port = intel_dig_port->port;
 	int type;
 
+	/* Initialize the work for modeset in case of link train failure */
+	INIT_WORK(&intel_connector->modeset_retry_work,
+		  intel_dp_modeset_retry_work_fn);
+
 	if (WARN(intel_dig_port->max_lanes < 1,
 		 "Not enough lanes (%d) for DP on port %c\n",
 		 intel_dig_port->max_lanes, port_name(port)))
 		return false;
 
+	intel_dp_set_source_rates(intel_dp);
+
 	intel_dp->reset_link_params = true;
 	intel_dp->pps_pipe = INVALID_PIPE;
 	intel_dp->active_pipe = INVALID_PIPE;
diff --git a/drivers/gpu/drm/i915/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
index 40ba3134545e..228ca06d9f0b 100644
--- a/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
@@ -28,6 +28,10 @@ static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable)
 {
 	uint8_t reg_val = 0;
 
+	/* Early return when display use other mechanism to enable backlight. */
+	if (!(intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_AUX_ENABLE_CAP))
+		return;
+
 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER,
 			      &reg_val) < 0) {
 		DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n",
@@ -74,8 +78,9 @@ static uint32_t intel_dp_aux_get_backlight(struct intel_connector *connector)
  * 8-bit or 16 bit value (MSB and LSB)
  */
 static void
-intel_dp_aux_set_backlight(struct intel_connector *connector, u32 level)
+intel_dp_aux_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
 	uint8_t vals[2] = { 0x0 };
 
@@ -93,24 +98,48 @@ intel_dp_aux_set_backlight(struct intel_connector *connector, u32 level)
 	}
 }
 
-static void intel_dp_aux_enable_backlight(struct intel_connector *connector)
+static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_state,
+					  const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
 	uint8_t dpcd_buf = 0;
+	uint8_t edp_backlight_mode = 0;
 
-	set_aux_backlight_enable(intel_dp, true);
+	if (drm_dp_dpcd_readb(&intel_dp->aux,
+			DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) != 1) {
+		DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n",
+			      DP_EDP_BACKLIGHT_MODE_SET_REGISTER);
+		return;
+	}
+
+	edp_backlight_mode = dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK;
+
+	switch (edp_backlight_mode) {
+	case DP_EDP_BACKLIGHT_CONTROL_MODE_PWM:
+	case DP_EDP_BACKLIGHT_CONTROL_MODE_PRESET:
+	case DP_EDP_BACKLIGHT_CONTROL_MODE_PRODUCT:
+		dpcd_buf &= ~DP_EDP_BACKLIGHT_CONTROL_MODE_MASK;
+		dpcd_buf |= DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD;
+		if (drm_dp_dpcd_writeb(&intel_dp->aux,
+			DP_EDP_BACKLIGHT_MODE_SET_REGISTER, dpcd_buf) < 0) {
+			DRM_DEBUG_KMS("Failed to write aux backlight mode\n");
+		}
+		break;
+
+	/* Do nothing when it is already DPCD mode */
+	case DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD:
+	default:
+		break;
+	}
 
-	if ((drm_dp_dpcd_readb(&intel_dp->aux,
-			       DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) == 1) &&
-	    ((dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK) ==
-	     DP_EDP_BACKLIGHT_CONTROL_MODE_PRESET))
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER,
-				   (dpcd_buf | DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD));
+	set_aux_backlight_enable(intel_dp, true);
+	intel_dp_aux_set_backlight(conn_state, connector->panel.backlight.level);
 }
 
-static void intel_dp_aux_disable_backlight(struct intel_connector *connector)
+static void intel_dp_aux_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
-	set_aux_backlight_enable(enc_to_intel_dp(&connector->encoder->base), false);
+	set_aux_backlight_enable(enc_to_intel_dp(old_conn_state->best_encoder), false);
 }
 
 static int intel_dp_aux_setup_backlight(struct intel_connector *connector,
@@ -137,13 +166,12 @@ intel_dp_aux_display_control_capable(struct intel_connector *connector)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
 
-	/* Check the  eDP Display control capabilities registers to determine if
+	/* Check the eDP Display control capabilities registers to determine if
 	 * the panel can support backlight control over the aux channel
 	 */
 	if (intel_dp->edp_dpcd[1] & DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP &&
-	    (intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_AUX_ENABLE_CAP) &&
-	    !((intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_PIN_ENABLE_CAP) ||
-	      (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_PWM_PIN_CAP))) {
+	    (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_AUX_SET_CAP) &&
+	    !(intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_PWM_PIN_CAP)) {
 		DRM_DEBUG_KMS("AUX Backlight Control Supported!\n");
 		return true;
 	}
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index 0048b520baf7..b79c1c0e404c 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -146,7 +146,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
 		link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
 	drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2);
 
-	if (intel_dp->num_sink_rates)
+	/* eDP 1.4 rate select method. */
+	if (!link_bw)
 		drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET,
 				  &rate_select, 1);
 
@@ -313,6 +314,24 @@ void intel_dp_stop_link_train(struct intel_dp *intel_dp)
 void
 intel_dp_start_link_train(struct intel_dp *intel_dp)
 {
-	intel_dp_link_training_clock_recovery(intel_dp);
-	intel_dp_link_training_channel_equalization(intel_dp);
+	struct intel_connector *intel_connector = intel_dp->attached_connector;
+
+	if (!intel_dp_link_training_clock_recovery(intel_dp))
+		goto failure_handling;
+	if (!intel_dp_link_training_channel_equalization(intel_dp))
+		goto failure_handling;
+
+	DRM_DEBUG_KMS("Link Training Passed at Link Rate = %d, Lane count = %d",
+		      intel_dp->link_rate, intel_dp->lane_count);
+	return;
+
+ failure_handling:
+	DRM_DEBUG_KMS("Link Training failed at link rate = %d, lane count = %d",
+		      intel_dp->link_rate, intel_dp->lane_count);
+	if (!intel_dp_get_link_train_fallback_values(intel_dp,
+						     intel_dp->link_rate,
+						     intel_dp->lane_count))
+		/* Schedule a Hotplug Uevent to userspace to start modeset */
+		schedule_work(&intel_connector->modeset_retry_work);
+	return;
 }
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 989e25577ac0..2cf046beae0f 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -39,7 +39,7 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
-	struct drm_atomic_state *state;
+	struct drm_atomic_state *state = pipe_config->base.state;
 	int bpp;
 	int lane_count, slots;
 	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
@@ -58,21 +58,26 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	 * for MST we always configure max link bw - the spec doesn't
 	 * seem to suggest we should do otherwise.
 	 */
-	lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
+	lane_count = intel_dp_max_lane_count(intel_dp);
 
 	pipe_config->lane_count = lane_count;
 
 	pipe_config->pipe_bpp = bpp;
-	pipe_config->port_clock = intel_dp_max_link_rate(intel_dp);
 
-	state = pipe_config->base.state;
+	pipe_config->port_clock = intel_dp_max_link_rate(intel_dp);
 
 	if (drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, connector->port))
 		pipe_config->has_audio = true;
-	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
 
+	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
 	pipe_config->pbn = mst_pbn;
-	slots = drm_dp_find_vcpi_slots(&intel_dp->mst_mgr, mst_pbn);
+
+	slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr,
+					      connector->port, mst_pbn);
+	if (slots < 0) {
+		DRM_DEBUG_KMS("failed finding vcpi slots:%d\n", slots);
+		return false;
+	}
 
 	intel_link_compute_m_n(bpp, lane_count,
 			       adjusted_mode->crtc_clock,
@@ -83,7 +88,38 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	pipe_config->dp_m_n.tu = slots;
 
 	return true;
+}
+
+static int intel_dp_mst_atomic_check(struct drm_connector *connector,
+		struct drm_connector_state *new_conn_state)
+{
+	struct drm_atomic_state *state = new_conn_state->state;
+	struct drm_connector_state *old_conn_state;
+	struct drm_crtc *old_crtc;
+	struct drm_crtc_state *crtc_state;
+	int slots, ret = 0;
+
+	old_conn_state = drm_atomic_get_old_connector_state(state, connector);
+	old_crtc = old_conn_state->crtc;
+	if (!old_crtc)
+		return ret;
+
+	crtc_state = drm_atomic_get_new_crtc_state(state, old_crtc);
+	slots = to_intel_crtc_state(crtc_state)->dp_m_n.tu;
+	if (drm_atomic_crtc_needs_modeset(crtc_state) && slots > 0) {
+		struct drm_dp_mst_topology_mgr *mgr;
+		struct drm_encoder *old_encoder;
+
+		old_encoder = old_conn_state->best_encoder;
+		mgr = &enc_to_mst(old_encoder)->primary->dp.mst_mgr;
 
+		ret = drm_dp_atomic_release_vcpi_slots(state, mgr, slots);
+		if (ret)
+			DRM_DEBUG_KMS("failed releasing %d vcpi slots:%d\n", slots, ret);
+		else
+			to_intel_crtc_state(crtc_state)->dp_m_n.tu = 0;
+	}
+	return ret;
 }
 
 static void intel_mst_disable_dp(struct intel_encoder *encoder,
@@ -297,14 +333,6 @@ intel_dp_mst_detect(struct drm_connector *connector, bool force)
 	return drm_dp_mst_detect_port(connector, &intel_dp->mst_mgr, intel_connector->port);
 }
 
-static int
-intel_dp_mst_set_property(struct drm_connector *connector,
-			  struct drm_property *property,
-			  uint64_t val)
-{
-	return 0;
-}
-
 static void
 intel_dp_mst_connector_destroy(struct drm_connector *connector)
 {
@@ -321,8 +349,7 @@ static const struct drm_connector_funcs intel_dp_mst_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_dp_mst_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = intel_dp_mst_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dp_mst_connector_destroy,
@@ -346,7 +373,7 @@ intel_dp_mst_mode_valid(struct drm_connector *connector,
 	int max_rate, mode_rate, max_lanes, max_link_clock;
 
 	max_link_clock = intel_dp_max_link_rate(intel_dp);
-	max_lanes = drm_dp_max_lane_count(intel_dp->dpcd);
+	max_lanes = intel_dp_max_lane_count(intel_dp);
 
 	max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes);
 	mode_rate = intel_dp_link_required(mode->clock, bpp);
@@ -390,6 +417,7 @@ static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_fun
 	.mode_valid = intel_dp_mst_mode_valid,
 	.atomic_best_encoder = intel_mst_atomic_best_encoder,
 	.best_encoder = intel_mst_best_encoder,
+	.atomic_check = intel_dp_mst_atomic_check,
 };
 
 static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder)
@@ -462,7 +490,6 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo
 		drm_mode_connector_attach_encoder(&intel_connector->base,
 						  &intel_dp->mst_encoders[i]->base.base);
 	}
-	intel_dp_add_properties(intel_dp, connector);
 
 	drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0);
 	drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0);
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index b4de632f1158..2f7b0e64f628 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -1321,7 +1321,6 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 	return true;
 }
 
-
 static bool
 skl_ddi_dp_set_dpll_hw_state(int clock,
 			     struct intel_dpll_hw_state *dpll_hw_state)
@@ -1967,6 +1966,438 @@ static const struct intel_dpll_mgr bxt_pll_mgr = {
 	.dump_hw_state = bxt_dump_hw_state,
 };
 
+static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
+			       struct intel_shared_dpll *pll)
+{
+	uint32_t val;
+
+	/* 1. Enable DPLL power in DPLL_ENABLE. */
+	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val |= PLL_POWER_ENABLE;
+	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+
+	/* 2. Wait for DPLL power state enabled in DPLL_ENABLE. */
+	if (intel_wait_for_register(dev_priv,
+				    CNL_DPLL_ENABLE(pll->id),
+				    PLL_POWER_STATE,
+				    PLL_POWER_STATE,
+				    5))
+		DRM_ERROR("PLL %d Power not enabled\n", pll->id);
+
+	/*
+	 * 3. Configure DPLL_CFGCR0 to set SSC enable/disable,
+	 * select DP mode, and set DP link rate.
+	 */
+	val = pll->state.hw_state.cfgcr0;
+	I915_WRITE(CNL_DPLL_CFGCR0(pll->id), val);
+
+	/* 4. Reab back to ensure writes completed */
+	POSTING_READ(CNL_DPLL_CFGCR0(pll->id));
+
+	/* 3. Configure DPLL_CFGCR0 */
+	/* Avoid touch CFGCR1 if HDMI mode is not enabled */
+	if (pll->state.hw_state.cfgcr0 & DPLL_CTRL1_HDMI_MODE(pll->id)) {
+		val = pll->state.hw_state.cfgcr1;
+		I915_WRITE(CNL_DPLL_CFGCR1(pll->id), val);
+		/* 4. Reab back to ensure writes completed */
+		POSTING_READ(CNL_DPLL_CFGCR1(pll->id));
+	}
+
+	/*
+	 * 5. If the frequency will result in a change to the voltage
+	 * requirement, follow the Display Voltage Frequency Switching
+	 * Sequence Before Frequency Change
+	 *
+	 * FIXME: (DVFS) is used to adjust the display voltage to match the
+	 * display clock frequencies
+	 */
+
+	/* 6. Enable DPLL in DPLL_ENABLE. */
+	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val |= PLL_ENABLE;
+	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+
+	/* 7. Wait for PLL lock status in DPLL_ENABLE. */
+	if (intel_wait_for_register(dev_priv,
+				    CNL_DPLL_ENABLE(pll->id),
+				    PLL_LOCK,
+				    PLL_LOCK,
+				    5))
+		DRM_ERROR("PLL %d not locked\n", pll->id);
+
+	/*
+	 * 8. If the frequency will result in a change to the voltage
+	 * requirement, follow the Display Voltage Frequency Switching
+	 * Sequence After Frequency Change
+	 *
+	 * FIXME: (DVFS) is used to adjust the display voltage to match the
+	 * display clock frequencies
+	 */
+
+	/*
+	 * 9. turn on the clock for the DDI and map the DPLL to the DDI
+	 * Done at intel_ddi_clk_select
+	 */
+}
+
+static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
+				struct intel_shared_dpll *pll)
+{
+	uint32_t val;
+
+	/*
+	 * 1. Configure DPCLKA_CFGCR0 to turn off the clock for the DDI.
+	 * Done at intel_ddi_post_disable
+	 */
+
+	/*
+	 * 2. If the frequency will result in a change to the voltage
+	 * requirement, follow the Display Voltage Frequency Switching
+	 * Sequence Before Frequency Change
+	 *
+	 * FIXME: (DVFS) is used to adjust the display voltage to match the
+	 * display clock frequencies
+	 */
+
+	/* 3. Disable DPLL through DPLL_ENABLE. */
+	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val &= ~PLL_ENABLE;
+	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+
+	/* 4. Wait for PLL not locked status in DPLL_ENABLE. */
+	if (intel_wait_for_register(dev_priv,
+				    CNL_DPLL_ENABLE(pll->id),
+				    PLL_LOCK,
+				    0,
+				    5))
+		DRM_ERROR("PLL %d locked\n", pll->id);
+
+	/*
+	 * 5. If the frequency will result in a change to the voltage
+	 * requirement, follow the Display Voltage Frequency Switching
+	 * Sequence After Frequency Change
+	 *
+	 * FIXME: (DVFS) is used to adjust the display voltage to match the
+	 * display clock frequencies
+	 */
+
+	/* 6. Disable DPLL power in DPLL_ENABLE. */
+	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val &= ~PLL_POWER_ENABLE;
+	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+
+	/* 7. Wait for DPLL power state disabled in DPLL_ENABLE. */
+	if (intel_wait_for_register(dev_priv,
+				    CNL_DPLL_ENABLE(pll->id),
+				    PLL_POWER_STATE,
+				    0,
+				    5))
+		DRM_ERROR("PLL %d Power not disabled\n", pll->id);
+}
+
+static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
+				     struct intel_shared_dpll *pll,
+				     struct intel_dpll_hw_state *hw_state)
+{
+	uint32_t val;
+	bool ret;
+
+	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
+		return false;
+
+	ret = false;
+
+	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	if (!(val & PLL_ENABLE))
+		goto out;
+
+	val = I915_READ(CNL_DPLL_CFGCR0(pll->id));
+	hw_state->cfgcr0 = val;
+
+	/* avoid reading back stale values if HDMI mode is not enabled */
+	if (val & DPLL_CFGCR0_HDMI_MODE) {
+		hw_state->cfgcr1 = I915_READ(CNL_DPLL_CFGCR1(pll->id));
+	}
+	ret = true;
+
+out:
+	intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
+
+	return ret;
+}
+
+static void cnl_wrpll_get_multipliers(unsigned int bestdiv,
+				      unsigned int *pdiv,
+				      unsigned int *qdiv,
+				      unsigned int *kdiv)
+{
+	/* even dividers */
+	if (bestdiv % 2 == 0) {
+		if (bestdiv == 2) {
+			*pdiv = 2;
+			*qdiv = 1;
+			*kdiv = 1;
+		} else if (bestdiv % 4 == 0) {
+			*pdiv = 2;
+			*qdiv = bestdiv / 4;
+			*kdiv = 2;
+		} else if (bestdiv % 6 == 0) {
+			*pdiv = 3;
+			*qdiv = bestdiv / 6;
+			*kdiv = 2;
+		} else if (bestdiv % 5 == 0) {
+			*pdiv = 5;
+			*qdiv = bestdiv / 10;
+			*kdiv = 2;
+		} else if (bestdiv % 14 == 0) {
+			*pdiv = 7;
+			*qdiv = bestdiv / 14;
+			*kdiv = 2;
+		}
+	} else {
+		if (bestdiv == 3 || bestdiv == 5 || bestdiv == 7) {
+			*pdiv = bestdiv;
+			*qdiv = 1;
+			*kdiv = 1;
+		} else { /* 9, 15, 21 */
+			*pdiv = bestdiv / 3;
+			*qdiv = 1;
+			*kdiv = 3;
+		}
+	}
+}
+
+static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, uint32_t dco_freq,
+				      uint32_t ref_freq, uint32_t pdiv, uint32_t qdiv,
+				      uint32_t kdiv)
+{
+	switch (kdiv) {
+	case 1:
+		params->kdiv = 1;
+		break;
+	case 2:
+		params->kdiv = 2;
+		break;
+	case 3:
+		params->kdiv = 4;
+		break;
+	default:
+		WARN(1, "Incorrect KDiv\n");
+	}
+
+	switch (pdiv) {
+	case 2:
+		params->pdiv = 1;
+		break;
+	case 3:
+		params->pdiv = 2;
+		break;
+	case 5:
+		params->pdiv = 4;
+		break;
+	case 7:
+		params->pdiv = 8;
+		break;
+	default:
+		WARN(1, "Incorrect PDiv\n");
+	}
+
+	if (kdiv != 2)
+		qdiv = 1;
+
+	params->qdiv_ratio = qdiv;
+	params->qdiv_mode = (qdiv == 1) ? 0 : 1;
+
+	params->dco_integer = div_u64(dco_freq, ref_freq);
+	params->dco_fraction = div_u64((div_u64((uint64_t)dco_freq<<15, (uint64_t)ref_freq) -
+					((uint64_t)params->dco_integer<<15)) * 0x8000, 0x8000);
+}
+
+static bool
+cnl_ddi_calculate_wrpll(int clock /* in Hz */,
+			struct drm_i915_private *dev_priv,
+			struct skl_wrpll_params *wrpll_params)
+{
+	uint64_t afe_clock = clock * 5 / KHz(1); /* clocks in kHz */
+	unsigned int dco_min = 7998 * KHz(1);
+	unsigned int dco_max = 10000 * KHz(1);
+	unsigned int dco_mid = (dco_min + dco_max) / 2;
+
+	static const int dividers[] = {  2,  4,  6,  8, 10, 12,  14,  16,
+					 18, 20, 24, 28, 30, 32,  36,  40,
+					 42, 44, 48, 50, 52, 54,  56,  60,
+					 64, 66, 68, 70, 72, 76,  78,  80,
+					 84, 88, 90, 92, 96, 98, 100, 102,
+					  3,  5,  7,  9, 15, 21 };
+	unsigned int d, dco;
+	unsigned int dco_centrality = 0;
+	unsigned int best_dco_centrality = 999999;
+	unsigned int best_div = 0;
+	unsigned int best_dco = 0;
+	unsigned int pdiv = 0, qdiv = 0, kdiv = 0;
+
+	for (d = 0; d < ARRAY_SIZE(dividers); d++) {
+		dco = afe_clock * dividers[d];
+
+		if ((dco <= dco_max) && (dco >= dco_min)) {
+			dco_centrality = abs(dco - dco_mid);
+
+			if (dco_centrality < best_dco_centrality) {
+				best_dco_centrality = dco_centrality;
+				best_div = dividers[d];
+				best_dco = dco;
+			}
+		}
+	}
+
+	if (best_div == 0)
+		return false;
+
+	cnl_wrpll_get_multipliers(best_div, &pdiv, &qdiv, &kdiv);
+
+	cnl_wrpll_params_populate(wrpll_params, best_dco,
+				  dev_priv->cdclk.hw.ref, pdiv, qdiv, kdiv);
+
+	return true;
+}
+
+static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
+				      struct intel_crtc_state *crtc_state,
+				      int clock)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	uint32_t cfgcr0, cfgcr1;
+	struct skl_wrpll_params wrpll_params = { 0, };
+
+	cfgcr0 = DPLL_CFGCR0_HDMI_MODE;
+
+	if (!cnl_ddi_calculate_wrpll(clock * 1000, dev_priv, &wrpll_params))
+		return false;
+
+	cfgcr0 |= DPLL_CFGCR0_DCO_FRACTION(wrpll_params.dco_fraction) |
+		wrpll_params.dco_integer;
+
+	cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(wrpll_params.qdiv_ratio) |
+		DPLL_CFGCR1_QDIV_MODE(wrpll_params.qdiv_mode) |
+		DPLL_CFGCR1_KDIV(wrpll_params.kdiv) |
+		DPLL_CFGCR1_PDIV(wrpll_params.pdiv) |
+		wrpll_params.central_freq |
+		DPLL_CFGCR1_CENTRAL_FREQ;
+
+	memset(&crtc_state->dpll_hw_state, 0,
+	       sizeof(crtc_state->dpll_hw_state));
+
+	crtc_state->dpll_hw_state.cfgcr0 = cfgcr0;
+	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
+	return true;
+}
+
+static bool
+cnl_ddi_dp_set_dpll_hw_state(int clock,
+			     struct intel_dpll_hw_state *dpll_hw_state)
+{
+	uint32_t cfgcr0;
+
+	cfgcr0 = DPLL_CFGCR0_SSC_ENABLE;
+
+	switch (clock / 2) {
+	case 81000:
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_810;
+		break;
+	case 135000:
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1350;
+		break;
+	case 270000:
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_2700;
+		break;
+		/* eDP 1.4 rates */
+	case 162000:
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1620;
+		break;
+	case 108000:
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1080;
+		break;
+	case 216000:
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_2160;
+		break;
+	case 324000:
+		/* Some SKUs may require elevated I/O voltage to support this */
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_3240;
+		break;
+	case 405000:
+		/* Some SKUs may require elevated I/O voltage to support this */
+		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_4050;
+		break;
+	}
+
+	dpll_hw_state->cfgcr0 = cfgcr0;
+	return true;
+}
+
+static struct intel_shared_dpll *
+cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+	     struct intel_encoder *encoder)
+{
+	struct intel_shared_dpll *pll;
+	int clock = crtc_state->port_clock;
+	bool bret;
+	struct intel_dpll_hw_state dpll_hw_state;
+
+	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
+
+	if (encoder->type == INTEL_OUTPUT_HDMI) {
+		bret = cnl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
+		if (!bret) {
+			DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
+			return NULL;
+		}
+	} else if (encoder->type == INTEL_OUTPUT_DP ||
+		   encoder->type == INTEL_OUTPUT_DP_MST ||
+		   encoder->type == INTEL_OUTPUT_EDP) {
+		bret = cnl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
+		if (!bret) {
+			DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
+			return NULL;
+		}
+		crtc_state->dpll_hw_state = dpll_hw_state;
+	} else {
+		DRM_DEBUG_KMS("Skip DPLL setup for encoder %d\n",
+			      encoder->type);
+		return NULL;
+	}
+
+	pll = intel_find_shared_dpll(crtc, crtc_state,
+				     DPLL_ID_SKL_DPLL0,
+				     DPLL_ID_SKL_DPLL2);
+	if (!pll) {
+		DRM_DEBUG_KMS("No PLL selected\n");
+		return NULL;
+	}
+
+	intel_reference_shared_dpll(pll, crtc_state);
+
+	return pll;
+}
+
+static const struct intel_shared_dpll_funcs cnl_ddi_pll_funcs = {
+	.enable = cnl_ddi_pll_enable,
+	.disable = cnl_ddi_pll_disable,
+	.get_hw_state = cnl_ddi_pll_get_hw_state,
+};
+
+static const struct dpll_info cnl_plls[] = {
+	{ "DPLL 0", DPLL_ID_SKL_DPLL0, &cnl_ddi_pll_funcs, 0 },
+	{ "DPLL 1", DPLL_ID_SKL_DPLL1, &cnl_ddi_pll_funcs, 0 },
+	{ "DPLL 2", DPLL_ID_SKL_DPLL2, &cnl_ddi_pll_funcs, 0 },
+	{ NULL, -1, NULL, },
+};
+
+static const struct intel_dpll_mgr cnl_pll_mgr = {
+	.dpll_info = cnl_plls,
+	.get_dpll = cnl_get_dpll,
+	.dump_hw_state = skl_dump_hw_state,
+};
+
 /**
  * intel_shared_dpll_init - Initialize shared DPLLs
  * @dev: drm device
@@ -1980,7 +2411,9 @@ void intel_shared_dpll_init(struct drm_device *dev)
 	const struct dpll_info *dpll_info;
 	int i;
 
-	if (IS_GEN9_BC(dev_priv))
+	if (IS_CANNONLAKE(dev_priv))
+		dpll_mgr = &cnl_pll_mgr;
+	else if (IS_GEN9_BC(dev_priv))
 		dpll_mgr = &skl_pll_mgr;
 	else if (IS_GEN9_LP(dev_priv))
 		dpll_mgr = &bxt_pll_mgr;
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h
index f8d13a947c13..f24ccf443d25 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h
@@ -128,6 +128,10 @@ struct intel_dpll_hw_state {
 	/* HDMI only, 0 when used for DP */
 	uint32_t cfgcr1, cfgcr2;
 
+	/* cnl */
+	uint32_t cfgcr0;
+	/* CNL also uses cfgcr1 */
+
 	/* bxt */
 	uint32_t ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10,
 		 pcsdw12;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index f630c7af5020..d93efb49a2e2 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -88,7 +88,6 @@
 	int cpu, ret, timeout = (US) * 1000; \
 	u64 base; \
 	_WAIT_FOR_ATOMIC_CHECK(ATOMIC); \
-	BUILD_BUG_ON((US) > 50000); \
 	if (!(ATOMIC)) { \
 		preempt_disable(); \
 		cpu = smp_processor_id(); \
@@ -130,8 +129,14 @@
 	ret__; \
 })
 
-#define wait_for_atomic(COND, MS)	_wait_for_atomic((COND), (MS) * 1000, 1)
-#define wait_for_atomic_us(COND, US)	_wait_for_atomic((COND), (US), 1)
+#define wait_for_atomic_us(COND, US) \
+({ \
+	BUILD_BUG_ON(!__builtin_constant_p(US)); \
+	BUILD_BUG_ON((US) > 50000); \
+	_wait_for_atomic((COND), (US), 1); \
+})
+
+#define wait_for_atomic(COND, MS) wait_for_atomic_us((COND), (MS) * 1000)
 
 #define KHz(x) (1000 * (x))
 #define MHz(x) KHz(1000 * (x))
@@ -261,7 +266,6 @@ struct intel_encoder {
 struct intel_panel {
 	struct drm_display_mode *fixed_mode;
 	struct drm_display_mode *downclock_mode;
-	int fitting_mode;
 
 	/* backlight */
 	struct {
@@ -284,9 +288,10 @@ struct intel_panel {
 		/* Connector and platform specific backlight functions */
 		int (*setup)(struct intel_connector *connector, enum pipe pipe);
 		uint32_t (*get)(struct intel_connector *connector);
-		void (*set)(struct intel_connector *connector, uint32_t level);
-		void (*disable)(struct intel_connector *connector);
-		void (*enable)(struct intel_connector *connector);
+		void (*set)(const struct drm_connector_state *conn_state, uint32_t level);
+		void (*disable)(const struct drm_connector_state *conn_state);
+		void (*enable)(const struct intel_crtc_state *crtc_state,
+			       const struct drm_connector_state *conn_state);
 		uint32_t (*hz_to_pwm)(struct intel_connector *connector,
 				      uint32_t hz);
 		void (*power)(struct intel_connector *, bool enable);
@@ -321,8 +326,20 @@ struct intel_connector {
 	void *port; /* store this opaque as its illegal to dereference it */
 
 	struct intel_dp *mst_port;
+
+	/* Work struct to schedule a uevent on link train failure */
+	struct work_struct modeset_retry_work;
 };
 
+struct intel_digital_connector_state {
+	struct drm_connector_state base;
+
+	enum hdmi_force_audio force_audio;
+	int broadcast_rgb;
+};
+
+#define to_intel_digital_connector_state(x) container_of(x, struct intel_digital_connector_state, base)
+
 struct dpll {
 	/* given values */
 	int n;
@@ -504,8 +521,8 @@ enum vlv_wm_level {
 };
 
 struct vlv_wm_state {
-	struct vlv_pipe_wm wm[NUM_VLV_WM_LEVELS];
-	struct vlv_sr_wm sr[NUM_VLV_WM_LEVELS];
+	struct g4x_pipe_wm wm[NUM_VLV_WM_LEVELS];
+	struct g4x_sr_wm sr[NUM_VLV_WM_LEVELS];
 	uint8_t num_levels;
 	bool cxsr;
 };
@@ -514,6 +531,22 @@ struct vlv_fifo_state {
 	u16 plane[I915_MAX_PLANES];
 };
 
+enum g4x_wm_level {
+	G4X_WM_LEVEL_NORMAL,
+	G4X_WM_LEVEL_SR,
+	G4X_WM_LEVEL_HPLL,
+	NUM_G4X_WM_LEVELS,
+};
+
+struct g4x_wm_state {
+	struct g4x_pipe_wm wm;
+	struct g4x_sr_wm sr;
+	struct g4x_sr_wm hpll;
+	bool cxsr;
+	bool hpll_en;
+	bool fbc_en;
+};
+
 struct intel_crtc_wm_state {
 	union {
 		struct {
@@ -541,7 +574,7 @@ struct intel_crtc_wm_state {
 
 		struct {
 			/* "raw" watermarks (not inverted) */
-			struct vlv_pipe_wm raw[NUM_VLV_WM_LEVELS];
+			struct g4x_pipe_wm raw[NUM_VLV_WM_LEVELS];
 			/* intermediate watermarks (inverted) */
 			struct vlv_wm_state intermediate;
 			/* optimal watermarks (inverted) */
@@ -549,6 +582,15 @@ struct intel_crtc_wm_state {
 			/* display FIFO split */
 			struct vlv_fifo_state fifo_state;
 		} vlv;
+
+		struct {
+			/* "raw" watermarks */
+			struct g4x_pipe_wm raw[NUM_G4X_WM_LEVELS];
+			/* intermediate watermarks */
+			struct g4x_wm_state intermediate;
+			/* optimal watermarks */
+			struct g4x_wm_state optimal;
+		} g4x;
 	};
 
 	/*
@@ -766,11 +808,6 @@ struct intel_crtc {
 	int adjusted_x;
 	int adjusted_y;
 
-	uint32_t cursor_addr;
-	uint32_t cursor_cntl;
-	uint32_t cursor_size;
-	uint32_t cursor_base;
-
 	struct intel_crtc_state *config;
 
 	/* global reset count when the last flip was submitted */
@@ -786,6 +823,7 @@ struct intel_crtc {
 		union {
 			struct intel_pipe_wm ilk;
 			struct vlv_wm_state vlv;
+			struct g4x_wm_state g4x;
 		} active;
 	} wm;
 
@@ -811,18 +849,22 @@ struct intel_plane {
 	int max_downscale;
 	uint32_t frontbuffer_bit;
 
+	struct {
+		u32 base, cntl, size;
+	} cursor;
+
 	/*
 	 * NOTE: Do not place new plane state fields here (e.g., when adding
 	 * new plane properties).  New runtime state should now be placed in
 	 * the intel_plane_state structure and accessed via plane_state.
 	 */
 
-	void (*update_plane)(struct drm_plane *plane,
+	void (*update_plane)(struct intel_plane *plane,
 			     const struct intel_crtc_state *crtc_state,
 			     const struct intel_plane_state *plane_state);
-	void (*disable_plane)(struct drm_plane *plane,
-			      struct drm_crtc *crtc);
-	int (*check_plane)(struct drm_plane *plane,
+	void (*disable_plane)(struct intel_plane *plane,
+			      struct intel_crtc *crtc);
+	int (*check_plane)(struct intel_plane *plane,
 			   struct intel_crtc_state *crtc_state,
 			   struct intel_plane_state *state);
 };
@@ -863,13 +905,9 @@ struct intel_hdmi {
 		enum drm_dp_dual_mode_type type;
 		int max_tmds_clock;
 	} dp_dual_mode;
-	bool limited_color_range;
-	bool color_range_auto;
 	bool has_hdmi_sink;
 	bool has_audio;
-	enum hdmi_force_audio force_audio;
 	bool rgb_quant_range_selectable;
-	enum hdmi_picture_aspect aspect_ratio;
 	struct intel_connector *attached_connector;
 	void (*write_infoframe)(struct drm_encoder *encoder,
 				const struct intel_crtc_state *crtc_state,
@@ -934,20 +972,24 @@ struct intel_dp {
 	bool detect_done;
 	bool channel_eq_status;
 	bool reset_link_params;
-	enum hdmi_force_audio force_audio;
-	bool limited_color_range;
-	bool color_range_auto;
 	uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
 	uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
 	uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
 	uint8_t edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE];
-	/* sink rates as reported by DP_SUPPORTED_LINK_RATES */
-	uint8_t num_sink_rates;
+	/* source rates */
+	int num_source_rates;
+	const int *source_rates;
+	/* sink rates as reported by DP_MAX_LINK_RATE/DP_SUPPORTED_LINK_RATES */
+	int num_sink_rates;
 	int sink_rates[DP_MAX_SUPPORTED_RATES];
-	/* Max lane count for the sink as per DPCD registers */
-	uint8_t max_sink_lane_count;
-	/* Max link BW for the sink as per DPCD registers */
-	int max_sink_link_bw;
+	bool use_rate_select;
+	/* intersection of source and sink rates */
+	int num_common_rates;
+	int common_rates[DP_MAX_SUPPORTED_RATES];
+	/* Max lane count for the current link */
+	int max_link_lane_count;
+	/* Max rate for the current link */
+	int max_link_rate;
 	/* sink or branch descriptor */
 	struct drm_dp_desc desc;
 	struct drm_dp_aux aux;
@@ -1268,6 +1310,8 @@ void intel_audio_deinit(struct drm_i915_private *dev_priv);
 /* intel_cdclk.c */
 void skl_init_cdclk(struct drm_i915_private *dev_priv);
 void skl_uninit_cdclk(struct drm_i915_private *dev_priv);
+void cnl_init_cdclk(struct drm_i915_private *dev_priv);
+void cnl_uninit_cdclk(struct drm_i915_private *dev_priv);
 void bxt_init_cdclk(struct drm_i915_private *dev_priv);
 void bxt_uninit_cdclk(struct drm_i915_private *dev_priv);
 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv);
@@ -1280,6 +1324,8 @@ void intel_set_cdclk(struct drm_i915_private *dev_priv,
 		     const struct intel_cdclk_state *cdclk_state);
 
 /* intel_display.c */
+void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
+void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
 enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv);
@@ -1300,7 +1346,6 @@ unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info
 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv);
 void intel_mark_busy(struct drm_i915_private *dev_priv);
 void intel_mark_idle(struct drm_i915_private *dev_priv);
-void intel_crtc_restore_mode(struct drm_crtc *crtc);
 int intel_display_suspend(struct drm_device *dev);
 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv);
 void intel_encoder_destroy(struct drm_encoder *encoder);
@@ -1479,15 +1524,16 @@ bool intel_dp_compute_config(struct intel_encoder *encoder,
 bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port);
 enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port,
 				  bool long_hpd);
-void intel_edp_backlight_on(struct intel_dp *intel_dp);
-void intel_edp_backlight_off(struct intel_dp *intel_dp);
+void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
+			    const struct drm_connector_state *conn_state);
+void intel_edp_backlight_off(const struct drm_connector_state *conn_state);
 void intel_edp_panel_vdd_on(struct intel_dp *intel_dp);
 void intel_edp_panel_on(struct intel_dp *intel_dp);
 void intel_edp_panel_off(struct intel_dp *intel_dp);
-void intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connector);
 void intel_dp_mst_suspend(struct drm_device *dev);
 void intel_dp_mst_resume(struct drm_device *dev);
 int intel_dp_max_link_rate(struct intel_dp *intel_dp);
+int intel_dp_max_lane_count(struct intel_dp *intel_dp);
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
 void intel_dp_hot_plug(struct intel_encoder *intel_encoder);
 void intel_power_sequencer_reset(struct drm_i915_private *dev_priv);
@@ -1660,12 +1706,13 @@ void intel_pch_panel_fitting(struct intel_crtc *crtc,
 void intel_gmch_panel_fitting(struct intel_crtc *crtc,
 			      struct intel_crtc_state *pipe_config,
 			      int fitting_mode);
-void intel_panel_set_backlight_acpi(struct intel_connector *connector,
+void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state,
 				    u32 level, u32 max);
 int intel_panel_setup_backlight(struct drm_connector *connector,
 				enum pipe pipe);
-void intel_panel_enable_backlight(struct intel_connector *connector);
-void intel_panel_disable_backlight(struct intel_connector *connector);
+void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state);
+void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state);
 void intel_panel_destroy_backlight(struct drm_connector *connector);
 enum drm_connector_status intel_panel_detect(struct drm_i915_private *dev_priv);
 extern struct drm_display_mode *intel_find_panel_downclock(
@@ -1815,6 +1862,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
 		    struct intel_rps_client *rps,
 		    unsigned long submitted);
 void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req);
+void g4x_wm_get_hw_state(struct drm_device *dev);
 void vlv_wm_get_hw_state(struct drm_device *dev);
 void ilk_wm_get_hw_state(struct drm_device *dev);
 void skl_wm_get_hw_state(struct drm_device *dev);
@@ -1822,6 +1870,7 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 			  struct skl_ddb_allocation *ddb /* out */);
 void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
 			      struct skl_pipe_wm *out);
+void g4x_wm_sanitize(struct drm_i915_private *dev_priv);
 void vlv_wm_sanitize(struct drm_i915_private *dev_priv);
 bool intel_can_enable_sagv(struct drm_atomic_state *state);
 int intel_enable_sagv(struct drm_i915_private *dev_priv);
@@ -1833,6 +1882,8 @@ bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries,
 				 int ignore);
 bool ilk_disable_lp_wm(struct drm_device *dev);
 int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
+int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
+				  struct intel_crtc_state *cstate);
 static inline int intel_enable_rc6(void)
 {
 	return i915.enable_rc6;
@@ -1857,10 +1908,19 @@ void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work
 void intel_tv_init(struct drm_i915_private *dev_priv);
 
 /* intel_atomic.c */
-int intel_connector_atomic_get_property(struct drm_connector *connector,
-					const struct drm_connector_state *state,
-					struct drm_property *property,
-					uint64_t *val);
+int intel_digital_connector_atomic_get_property(struct drm_connector *connector,
+						const struct drm_connector_state *state,
+						struct drm_property *property,
+						uint64_t *val);
+int intel_digital_connector_atomic_set_property(struct drm_connector *connector,
+						struct drm_connector_state *state,
+						struct drm_property *property,
+						uint64_t val);
+int intel_digital_connector_atomic_check(struct drm_connector *conn,
+					 struct drm_connector_state *new_state);
+struct drm_connector_state *
+intel_digital_connector_duplicate_state(struct drm_connector *connector);
+
 struct drm_crtc_state *intel_crtc_duplicate_state(struct drm_crtc *crtc);
 void intel_crtc_destroy_state(struct drm_crtc *crtc,
 			       struct drm_crtc_state *state);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index fc0ef492252a..50ec836da8b1 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -320,10 +320,10 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder,
 
 		if (HAS_GMCH_DISPLAY(dev_priv))
 			intel_gmch_panel_fitting(crtc, pipe_config,
-						 intel_connector->panel.fitting_mode);
+						 conn_state->scaling_mode);
 		else
 			intel_pch_panel_fitting(crtc, pipe_config,
-						intel_connector->panel.fitting_mode);
+						conn_state->scaling_mode);
 	}
 
 	/* DSI uses short packets for sync events, so clear mode flags for DSI */
@@ -346,12 +346,13 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder,
 	return true;
 }
 
-static void glk_dsi_device_ready(struct intel_encoder *encoder)
+static bool glk_dsi_enable_io(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
-	u32 tmp, val;
+	u32 tmp;
+	bool cold_boot = false;
 
 	/* Set the MIPI mode
 	 * If MIPI_Mode is off, then writing to LP_Wake bit is not reflecting.
@@ -370,7 +371,10 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 	/* Program LP Wake */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		tmp = I915_READ(MIPI_CTRL(port));
-		tmp |= GLK_LP_WAKE;
+		if (!(I915_READ(MIPI_DEVICE_READY(port)) & DEVICE_READY))
+			tmp &= ~GLK_LP_WAKE;
+		else
+			tmp |= GLK_LP_WAKE;
 		I915_WRITE(MIPI_CTRL(port), tmp);
 	}
 
@@ -382,6 +386,22 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 			DRM_ERROR("MIPIO port is powergated\n");
 	}
 
+	/* Check for cold boot scenario */
+	for_each_dsi_port(port, intel_dsi->ports) {
+		cold_boot |= !(I915_READ(MIPI_DEVICE_READY(port)) &
+							DEVICE_READY);
+	}
+
+	return cold_boot;
+}
+
+static void glk_dsi_device_ready(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+	enum port port;
+	u32 val;
+
 	/* Wait for MIPI PHY status bit to set */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		if (intel_wait_for_register(dev_priv,
@@ -391,8 +411,8 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 	}
 
 	/* Get IO out of reset */
-	tmp = I915_READ(MIPI_CTRL(PORT_A));
-	I915_WRITE(MIPI_CTRL(PORT_A), tmp | GLK_MIPIIO_RESET_RELEASED);
+	val = I915_READ(MIPI_CTRL(PORT_A));
+	I915_WRITE(MIPI_CTRL(PORT_A), val | GLK_MIPIIO_RESET_RELEASED);
 
 	/* Get IO out of Low power state*/
 	for_each_dsi_port(port, intel_dsi->ports) {
@@ -402,34 +422,34 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 			val |= DEVICE_READY;
 			I915_WRITE(MIPI_DEVICE_READY(port), val);
 			usleep_range(10, 15);
-		}
-
-		/* Enter ULPS */
-		val = I915_READ(MIPI_DEVICE_READY(port));
-		val &= ~ULPS_STATE_MASK;
-		val |= (ULPS_STATE_ENTER | DEVICE_READY);
-		I915_WRITE(MIPI_DEVICE_READY(port), val);
+		} else {
+			/* Enter ULPS */
+			val = I915_READ(MIPI_DEVICE_READY(port));
+			val &= ~ULPS_STATE_MASK;
+			val |= (ULPS_STATE_ENTER | DEVICE_READY);
+			I915_WRITE(MIPI_DEVICE_READY(port), val);
 
-		/* Wait for ULPS active */
-		if (intel_wait_for_register(dev_priv,
+			/* Wait for ULPS active */
+			if (intel_wait_for_register(dev_priv,
 				MIPI_CTRL(port), GLK_ULPS_NOT_ACTIVE, 0, 20))
-			DRM_ERROR("ULPS not active\n");
+				DRM_ERROR("ULPS not active\n");
 
-		/* Exit ULPS */
-		val = I915_READ(MIPI_DEVICE_READY(port));
-		val &= ~ULPS_STATE_MASK;
-		val |= (ULPS_STATE_EXIT | DEVICE_READY);
-		I915_WRITE(MIPI_DEVICE_READY(port), val);
+			/* Exit ULPS */
+			val = I915_READ(MIPI_DEVICE_READY(port));
+			val &= ~ULPS_STATE_MASK;
+			val |= (ULPS_STATE_EXIT | DEVICE_READY);
+			I915_WRITE(MIPI_DEVICE_READY(port), val);
 
-		/* Enter Normal Mode */
-		val = I915_READ(MIPI_DEVICE_READY(port));
-		val &= ~ULPS_STATE_MASK;
-		val |= (ULPS_STATE_NORMAL_OPERATION | DEVICE_READY);
-		I915_WRITE(MIPI_DEVICE_READY(port), val);
+			/* Enter Normal Mode */
+			val = I915_READ(MIPI_DEVICE_READY(port));
+			val &= ~ULPS_STATE_MASK;
+			val |= (ULPS_STATE_NORMAL_OPERATION | DEVICE_READY);
+			I915_WRITE(MIPI_DEVICE_READY(port), val);
 
-		tmp = I915_READ(MIPI_CTRL(port));
-		tmp &= ~GLK_LP_WAKE;
-		I915_WRITE(MIPI_CTRL(port), tmp);
+			val = I915_READ(MIPI_CTRL(port));
+			val &= ~GLK_LP_WAKE;
+			I915_WRITE(MIPI_CTRL(port), val);
+		}
 	}
 
 	/* Wait for Stop state */
@@ -770,6 +790,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
 	u32 val;
+	bool glk_cold_boot = false;
 
 	DRM_DEBUG_KMS("\n");
 
@@ -800,7 +821,8 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 		I915_WRITE(DSPCLK_GATE_D, val);
 	}
 
-	intel_dsi_prepare(encoder, pipe_config);
+	if (!IS_GEMINILAKE(dev_priv))
+		intel_dsi_prepare(encoder, pipe_config);
 
 	/* Power on, try both CRC pmic gpio and VBT */
 	if (intel_dsi->gpio_panel)
@@ -811,9 +833,21 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 	/* Deassert reset */
 	intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DEASSERT_RESET);
 
+	if (IS_GEMINILAKE(dev_priv)) {
+		glk_cold_boot = glk_dsi_enable_io(encoder);
+
+		/* Prepare port in cold boot(s3/s4) scenario */
+		if (glk_cold_boot)
+			intel_dsi_prepare(encoder, pipe_config);
+	}
+
 	/* Put device in ready state (LP-11) */
 	intel_dsi_device_ready(encoder);
 
+	/* Prepare port in normal boot scenario */
+	if (IS_GEMINILAKE(dev_priv) && !glk_cold_boot)
+		intel_dsi_prepare(encoder, pipe_config);
+
 	/* Send initialization commands in LP mode */
 	intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_INIT_OTP);
 
@@ -835,7 +869,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 		intel_dsi_port_enable(encoder);
 	}
 
-	intel_panel_enable_backlight(intel_dsi->attached_connector);
+	intel_panel_enable_backlight(pipe_config, conn_state);
 	intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_ON);
 }
 
@@ -866,7 +900,7 @@ static void intel_dsi_disable(struct intel_encoder *encoder,
 	DRM_DEBUG_KMS("\n");
 
 	intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_OFF);
-	intel_panel_disable_backlight(intel_dsi->attached_connector);
+	intel_panel_disable_backlight(old_conn_state);
 
 	/*
 	 * Disable Device ready before the port shutdown in order
@@ -1587,48 +1621,6 @@ static int intel_dsi_get_modes(struct drm_connector *connector)
 	return 1;
 }
 
-static int intel_dsi_set_property(struct drm_connector *connector,
-				  struct drm_property *property,
-				  uint64_t val)
-{
-	struct drm_device *dev = connector->dev;
-	struct intel_connector *intel_connector = to_intel_connector(connector);
-	struct drm_crtc *crtc;
-	int ret;
-
-	ret = drm_object_property_set_value(&connector->base, property, val);
-	if (ret)
-		return ret;
-
-	if (property == dev->mode_config.scaling_mode_property) {
-		if (val == DRM_MODE_SCALE_NONE) {
-			DRM_DEBUG_KMS("no scaling not supported\n");
-			return -EINVAL;
-		}
-		if (HAS_GMCH_DISPLAY(to_i915(dev)) &&
-		    val == DRM_MODE_SCALE_CENTER) {
-			DRM_DEBUG_KMS("centering not supported\n");
-			return -EINVAL;
-		}
-
-		if (intel_connector->panel.fitting_mode == val)
-			return 0;
-
-		intel_connector->panel.fitting_mode = val;
-	}
-
-	crtc = connector->state->crtc;
-	if (crtc && crtc->state->enable) {
-		/*
-		 * If the CRTC is enabled, the display will be changed
-		 * according to the new panel fitting mode.
-		 */
-		intel_crtc_restore_mode(crtc);
-	}
-
-	return 0;
-}
-
 static void intel_dsi_connector_destroy(struct drm_connector *connector)
 {
 	struct intel_connector *intel_connector = to_intel_connector(connector);
@@ -1657,6 +1649,7 @@ static const struct drm_encoder_funcs intel_dsi_funcs = {
 static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs = {
 	.get_modes = intel_dsi_get_modes,
 	.mode_valid = intel_dsi_mode_valid,
+	.atomic_check = intel_digital_connector_atomic_check,
 };
 
 static const struct drm_connector_funcs intel_dsi_connector_funcs = {
@@ -1665,22 +1658,28 @@ static const struct drm_connector_funcs intel_dsi_connector_funcs = {
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dsi_connector_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = intel_dsi_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.atomic_get_property = intel_digital_connector_atomic_get_property,
+	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
 };
 
 static void intel_dsi_add_properties(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 
 	if (connector->panel.fixed_mode) {
-		drm_mode_create_scaling_mode_property(dev);
-		drm_object_attach_property(&connector->base.base,
-					   dev->mode_config.scaling_mode_property,
-					   DRM_MODE_SCALE_ASPECT);
-		connector->panel.fitting_mode = DRM_MODE_SCALE_ASPECT;
+		u32 allowed_scalers;
+
+		allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN);
+		if (!HAS_GMCH_DISPLAY(dev_priv))
+			allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER);
+
+		drm_connector_attach_scaling_mode_property(&connector->base,
+								allowed_scalers);
+
+		connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT;
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
index ac7c6020c443..6e09ceb71500 100644
--- a/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
+++ b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
@@ -60,10 +60,9 @@ static u32 dcs_get_backlight(struct intel_connector *connector)
 	return data;
 }
 
-static void dcs_set_backlight(struct intel_connector *connector, u32 level)
+static void dcs_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
-	struct intel_encoder *encoder = connector->encoder;
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(conn_state->best_encoder);
 	struct mipi_dsi_device *dsi_device;
 	u8 data = level;
 	enum port port;
@@ -76,14 +75,13 @@ static void dcs_set_backlight(struct intel_connector *connector, u32 level)
 	}
 }
 
-static void dcs_disable_backlight(struct intel_connector *connector)
+static void dcs_disable_backlight(const struct drm_connector_state *conn_state)
 {
-	struct intel_encoder *encoder = connector->encoder;
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(conn_state->best_encoder);
 	struct mipi_dsi_device *dsi_device;
 	enum port port;
 
-	dcs_set_backlight(connector, 0);
+	dcs_set_backlight(conn_state, 0);
 
 	for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) {
 		u8 cabc = POWER_SAVE_OFF;
@@ -110,11 +108,11 @@ static void dcs_disable_backlight(struct intel_connector *connector)
 	}
 }
 
-static void dcs_enable_backlight(struct intel_connector *connector)
+static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
 {
-	struct intel_encoder *encoder = connector->encoder;
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
-	struct intel_panel *panel = &connector->panel;
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(conn_state->best_encoder);
+	struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel;
 	struct mipi_dsi_device *dsi_device;
 	enum port port;
 
@@ -142,7 +140,7 @@ static void dcs_enable_backlight(struct intel_connector *connector)
 				   &cabc, sizeof(cabc));
 	}
 
-	dcs_set_backlight(connector, panel->backlight.level);
+	dcs_set_backlight(conn_state, panel->backlight.level);
 }
 
 static int dcs_setup_backlight(struct intel_connector *connector,
diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c
index 0dce7792643a..7158c7ce9c09 100644
--- a/drivers/gpu/drm/i915/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c
@@ -694,8 +694,8 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id)
 						clk_zero_cnt << 8 | prepare_cnt;
 
 	/*
-	 * LP to HS switch count = 4TLPX + PREP_COUNT * 2 + EXIT_ZERO_COUNT * 2
-	 *					+ 10UI + Extra Byte Count
+	 * LP to HS switch count = 4TLPX + PREP_COUNT * mul + EXIT_ZERO_COUNT *
+	 *					mul + 10UI + Extra Byte Count
 	 *
 	 * HS to LP switch count = THS-TRAIL + 2TLPX + Extra Byte Count
 	 * Extra Byte Count is calculated according to number of lanes.
@@ -708,8 +708,8 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id)
 	/* B044 */
 	/* FIXME:
 	 * The comment above does not match with the code */
-	lp_to_hs_switch = DIV_ROUND_UP(4 * tlpx_ui + prepare_cnt * 2 +
-						exit_zero_cnt * 2 + 10, 8);
+	lp_to_hs_switch = DIV_ROUND_UP(4 * tlpx_ui + prepare_cnt * mul +
+						exit_zero_cnt * mul + 10, 8);
 
 	hs_to_lp_switch = DIV_ROUND_UP(mipi_config->ths_trail + 2 * tlpx_ui, 8);
 
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 6025839ed3b7..c1544a53095d 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -350,7 +350,7 @@ static const struct drm_connector_funcs intel_dvo_connector_funcs = {
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dvo_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 };
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index f94eacff196c..a4487c5b7e37 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -26,69 +26,177 @@
 #include "intel_ringbuffer.h"
 #include "intel_lrc.h"
 
-static const struct engine_info {
+/* Haswell does have the CXT_SIZE register however it does not appear to be
+ * valid. Now, docs explain in dwords what is in the context object. The full
+ * size is 70720 bytes, however, the power context and execlist context will
+ * never be saved (power context is stored elsewhere, and execlists don't work
+ * on HSW) - so the final size, including the extra state required for the
+ * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
+ */
+#define HSW_CXT_TOTAL_SIZE		(17 * PAGE_SIZE)
+/* Same as Haswell, but 72064 bytes now. */
+#define GEN8_CXT_TOTAL_SIZE		(18 * PAGE_SIZE)
+
+#define GEN8_LR_CONTEXT_RENDER_SIZE	(20 * PAGE_SIZE)
+#define GEN9_LR_CONTEXT_RENDER_SIZE	(22 * PAGE_SIZE)
+
+#define GEN8_LR_CONTEXT_OTHER_SIZE	( 2 * PAGE_SIZE)
+
+struct engine_class_info {
 	const char *name;
-	unsigned int exec_id;
+	int (*init_legacy)(struct intel_engine_cs *engine);
+	int (*init_execlists)(struct intel_engine_cs *engine);
+};
+
+static const struct engine_class_info intel_engine_classes[] = {
+	[RENDER_CLASS] = {
+		.name = "rcs",
+		.init_execlists = logical_render_ring_init,
+		.init_legacy = intel_init_render_ring_buffer,
+	},
+	[COPY_ENGINE_CLASS] = {
+		.name = "bcs",
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_blt_ring_buffer,
+	},
+	[VIDEO_DECODE_CLASS] = {
+		.name = "vcs",
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_bsd_ring_buffer,
+	},
+	[VIDEO_ENHANCEMENT_CLASS] = {
+		.name = "vecs",
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_vebox_ring_buffer,
+	},
+};
+
+struct engine_info {
 	unsigned int hw_id;
+	unsigned int uabi_id;
+	u8 class;
+	u8 instance;
 	u32 mmio_base;
 	unsigned irq_shift;
-	int (*init_legacy)(struct intel_engine_cs *engine);
-	int (*init_execlists)(struct intel_engine_cs *engine);
-} intel_engines[] = {
+};
+
+static const struct engine_info intel_engines[] = {
 	[RCS] = {
-		.name = "rcs",
 		.hw_id = RCS_HW,
-		.exec_id = I915_EXEC_RENDER,
+		.uabi_id = I915_EXEC_RENDER,
+		.class = RENDER_CLASS,
+		.instance = 0,
 		.mmio_base = RENDER_RING_BASE,
 		.irq_shift = GEN8_RCS_IRQ_SHIFT,
-		.init_execlists = logical_render_ring_init,
-		.init_legacy = intel_init_render_ring_buffer,
 	},
 	[BCS] = {
-		.name = "bcs",
 		.hw_id = BCS_HW,
-		.exec_id = I915_EXEC_BLT,
+		.uabi_id = I915_EXEC_BLT,
+		.class = COPY_ENGINE_CLASS,
+		.instance = 0,
 		.mmio_base = BLT_RING_BASE,
 		.irq_shift = GEN8_BCS_IRQ_SHIFT,
-		.init_execlists = logical_xcs_ring_init,
-		.init_legacy = intel_init_blt_ring_buffer,
 	},
 	[VCS] = {
-		.name = "vcs",
 		.hw_id = VCS_HW,
-		.exec_id = I915_EXEC_BSD,
+		.uabi_id = I915_EXEC_BSD,
+		.class = VIDEO_DECODE_CLASS,
+		.instance = 0,
 		.mmio_base = GEN6_BSD_RING_BASE,
 		.irq_shift = GEN8_VCS1_IRQ_SHIFT,
-		.init_execlists = logical_xcs_ring_init,
-		.init_legacy = intel_init_bsd_ring_buffer,
 	},
 	[VCS2] = {
-		.name = "vcs2",
 		.hw_id = VCS2_HW,
-		.exec_id = I915_EXEC_BSD,
+		.uabi_id = I915_EXEC_BSD,
+		.class = VIDEO_DECODE_CLASS,
+		.instance = 1,
 		.mmio_base = GEN8_BSD2_RING_BASE,
 		.irq_shift = GEN8_VCS2_IRQ_SHIFT,
-		.init_execlists = logical_xcs_ring_init,
-		.init_legacy = intel_init_bsd2_ring_buffer,
 	},
 	[VECS] = {
-		.name = "vecs",
 		.hw_id = VECS_HW,
-		.exec_id = I915_EXEC_VEBOX,
+		.uabi_id = I915_EXEC_VEBOX,
+		.class = VIDEO_ENHANCEMENT_CLASS,
+		.instance = 0,
 		.mmio_base = VEBOX_RING_BASE,
 		.irq_shift = GEN8_VECS_IRQ_SHIFT,
-		.init_execlists = logical_xcs_ring_init,
-		.init_legacy = intel_init_vebox_ring_buffer,
 	},
 };
 
+/**
+ * ___intel_engine_context_size() - return the size of the context for an engine
+ * @dev_priv: i915 device private
+ * @class: engine class
+ *
+ * Each engine class may require a different amount of space for a context
+ * image.
+ *
+ * Return: size (in bytes) of an engine class specific context image
+ *
+ * Note: this size includes the HWSP, which is part of the context image
+ * in LRC mode, but does not include the "shared data page" used with
+ * GuC submission. The caller should account for this if using the GuC.
+ */
+static u32
+__intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
+{
+	u32 cxt_size;
+
+	BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
+
+	switch (class) {
+	case RENDER_CLASS:
+		switch (INTEL_GEN(dev_priv)) {
+		default:
+			MISSING_CASE(INTEL_GEN(dev_priv));
+		case 9:
+			return GEN9_LR_CONTEXT_RENDER_SIZE;
+		case 8:
+			return i915.enable_execlists ?
+			       GEN8_LR_CONTEXT_RENDER_SIZE :
+			       GEN8_CXT_TOTAL_SIZE;
+		case 7:
+			if (IS_HASWELL(dev_priv))
+				return HSW_CXT_TOTAL_SIZE;
+
+			cxt_size = I915_READ(GEN7_CXT_SIZE);
+			return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
+					PAGE_SIZE);
+		case 6:
+			cxt_size = I915_READ(CXT_SIZE);
+			return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
+					PAGE_SIZE);
+		case 5:
+		case 4:
+		case 3:
+		case 2:
+		/* For the special day when i810 gets merged. */
+		case 1:
+			return 0;
+		}
+		break;
+	default:
+		MISSING_CASE(class);
+	case VIDEO_DECODE_CLASS:
+	case VIDEO_ENHANCEMENT_CLASS:
+	case COPY_ENGINE_CLASS:
+		if (INTEL_GEN(dev_priv) < 8)
+			return 0;
+		return GEN8_LR_CONTEXT_OTHER_SIZE;
+	}
+}
+
 static int
 intel_engine_setup(struct drm_i915_private *dev_priv,
 		   enum intel_engine_id id)
 {
 	const struct engine_info *info = &intel_engines[id];
+	const struct engine_class_info *class_info;
 	struct intel_engine_cs *engine;
 
+	GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
+	class_info = &intel_engine_classes[info->class];
+
 	GEM_BUG_ON(dev_priv->engine[id]);
 	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 	if (!engine)
@@ -96,11 +204,20 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 
 	engine->id = id;
 	engine->i915 = dev_priv;
-	engine->name = info->name;
-	engine->exec_id = info->exec_id;
+	WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
+			 class_info->name, info->instance) >=
+		sizeof(engine->name));
+	engine->uabi_id = info->uabi_id;
 	engine->hw_id = engine->guc_id = info->hw_id;
 	engine->mmio_base = info->mmio_base;
 	engine->irq_shift = info->irq_shift;
+	engine->class = info->class;
+	engine->instance = info->instance;
+
+	engine->context_size = __intel_engine_context_size(dev_priv,
+							   engine->class);
+	if (WARN_ON(engine->context_size > BIT(20)))
+		engine->context_size = 0;
 
 	/* Nothing to do here, execute in order of dependencies */
 	engine->schedule = NULL;
@@ -112,18 +229,18 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 }
 
 /**
- * intel_engines_init_early() - allocate the Engine Command Streamers
+ * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
  * @dev_priv: i915 device private
  *
  * Return: non-zero if the initialization failed.
  */
-int intel_engines_init_early(struct drm_i915_private *dev_priv)
+int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 {
 	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
-	unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
-	unsigned int mask = 0;
+	const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
+	unsigned int mask = 0;
 	unsigned int i;
 	int err;
 
@@ -150,6 +267,12 @@ int intel_engines_init_early(struct drm_i915_private *dev_priv)
 	if (WARN_ON(mask != ring_mask))
 		device_info->ring_mask = mask;
 
+	/* We always presume we have at least RCS available for later probing */
+	if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
+		err = -ENODEV;
+		goto cleanup;
+	}
+
 	device_info->num_rings = hweight32(mask);
 
 	return 0;
@@ -161,7 +284,7 @@ cleanup:
 }
 
 /**
- * intel_engines_init() - allocate, populate and init the Engine Command Streamers
+ * intel_engines_init() - init the Engine Command Streamers
  * @dev_priv: i915 device private
  *
  * Return: non-zero if the initialization failed.
@@ -175,12 +298,14 @@ int intel_engines_init(struct drm_i915_private *dev_priv)
 	int err = 0;
 
 	for_each_engine(engine, dev_priv, id) {
+		const struct engine_class_info *class_info =
+			&intel_engine_classes[engine->class];
 		int (*init)(struct intel_engine_cs *engine);
 
 		if (i915.enable_execlists)
-			init = intel_engines[id].init_execlists;
+			init = class_info->init_execlists;
 		else
-			init = intel_engines[id].init_legacy;
+			init = class_info->init_legacy;
 		if (!init) {
 			kfree(engine);
 			dev_priv->engine[id] = NULL;
@@ -223,6 +348,9 @@ void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
+	GEM_BUG_ON(!intel_engine_is_idle(engine));
+	GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
+
 	/* Our semaphore implementation is strictly monotonic (i.e. we proceed
 	 * so long as the semaphore value in the register/page is greater
 	 * than the sync value), so whenever we reset the seqno,
@@ -253,13 +381,12 @@ void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
 	clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 
-	GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
-	engine->hangcheck.seqno = seqno;
-
 	/* After manually advancing the seqno, fake the interrupt in case
 	 * there are any waiters for that seqno.
 	 */
 	intel_engine_wakeup(engine);
+
+	GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
 }
 
 static void intel_engine_init_timeline(struct intel_engine_cs *engine)
@@ -342,6 +469,7 @@ static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
  */
 int intel_engine_init_common(struct intel_engine_cs *engine)
 {
+	struct intel_ring *ring;
 	int ret;
 
 	engine->set_default_submission(engine);
@@ -353,9 +481,9 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	 * be available. To avoid this we always pin the default
 	 * context.
 	 */
-	ret = engine->context_pin(engine, engine->i915->kernel_context);
-	if (ret)
-		return ret;
+	ring = engine->context_pin(engine, engine->i915->kernel_context);
+	if (IS_ERR(ring))
+		return PTR_ERR(ring);
 
 	ret = intel_engine_init_breadcrumbs(engine);
 	if (ret)
@@ -686,26 +814,27 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 	int ret;
 
-	/* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk */
+	/* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
 	I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
 
-	/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk */
+	/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
 	I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
 		   GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
 
-	/* WaDisableKillLogic:bxt,skl,kbl */
+	/* WaDisableKillLogic:bxt,skl,kbl,cfl */
 	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 		   ECOCHK_DIS_TLB);
 
-	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk */
-	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk */
+	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
+	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 			  FLOW_CONTROL_ENABLE |
 			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 
 	/* Syncing dependencies between camera and graphics:skl,bxt,kbl */
-	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
-			  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
+	if (!IS_COFFEELAKE(dev_priv))
+		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
+				  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
 	/* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
 	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
@@ -723,16 +852,18 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 		 */
 	}
 
-	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */
+	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
+	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
+			  GEN9_ENABLE_YV12_BUGFIX |
 			  GEN9_ENABLE_GPGPU_PREEMPTION);
 
-	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk */
-	/* WaDisablePartialResolveInVc:skl,bxt,kbl */
+	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
+	/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 	WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 					 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
 
-	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk */
+	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 			  GEN9_CCS_TLB_PREFETCH_ENABLE);
 
@@ -741,7 +872,7 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 		WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
 				  PIXEL_MASK_CAMMING_DISABLE);
 
-	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl */
+	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 	WA_SET_BIT_MASKED(HDC_CHICKEN0,
 			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 			  HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
@@ -759,39 +890,41 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 	 * a TLB invalidation occurs during a PSD flush.
 	 */
 
-	/* WaForceEnableNonCoherent:skl,bxt,kbl */
+	/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 	WA_SET_BIT_MASKED(HDC_CHICKEN0,
 			  HDC_FORCE_NON_COHERENT);
 
 	/* WaDisableHDCInvalidation:skl,bxt,kbl */
-	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
-		   BDW_DISABLE_HDC_INVALIDATION);
+	if (!IS_COFFEELAKE(dev_priv))
+		I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
+			   BDW_DISABLE_HDC_INVALIDATION);
 
-	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl */
+	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 	if (IS_SKYLAKE(dev_priv) ||
 	    IS_KABYLAKE(dev_priv) ||
+	    IS_COFFEELAKE(dev_priv) ||
 	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
 		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 				  GEN8_SAMPLER_POWER_BYPASS_DIS);
 
-	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk */
+	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 
-	/* WaOCLCoherentLineFlush:skl,bxt,kbl */
+	/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
 	I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
 				    GEN8_LQSC_FLUSH_COHERENT_LINES));
 
-	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk */
+	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
 	ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
 	if (ret)
 		return ret;
 
-	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl */
+	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl */
 	ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
 	if (ret)
 		return ret;
 
-	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk */
+	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
 	ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
 	if (ret)
 		return ret;
@@ -1010,6 +1143,38 @@ static int glk_init_workarounds(struct intel_engine_cs *engine)
 	return 0;
 }
 
+static int cfl_init_workarounds(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	int ret;
+
+	ret = gen9_init_workarounds(engine);
+	if (ret)
+		return ret;
+
+	/* WaEnableGapsTsvCreditFix:cfl */
+	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
+				   GEN9_GAPS_TSV_CREDIT_DISABLE));
+
+	/* WaToEnableHwFixForPushConstHWBug:cfl */
+	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+
+	/* WaDisableGafsUnitClkGating:cfl */
+	WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaDisableSbeCacheDispatchPortSharing:cfl */
+	WA_SET_BIT_MASKED(
+		GEN7_HALF_SLICE_CHICKEN1,
+		GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
+
+	/* WaInPlaceDecompressionHang:cfl */
+	WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
+		   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+
+	return 0;
+}
+
 int init_workarounds_ring(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -1032,6 +1197,8 @@ int init_workarounds_ring(struct intel_engine_cs *engine)
 		err = kbl_init_workarounds(engine);
 	else if (IS_GEMINILAKE(dev_priv))
 		err =  glk_init_workarounds(engine);
+	else if (IS_COFFEELAKE(dev_priv))
+		err = cfl_init_workarounds(engine);
 	else
 		err = 0;
 	if (err)
@@ -1082,6 +1249,11 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
 
 	intel_runtime_pm_get(dev_priv);
 
+	/* First check that no commands are left in the ring */
+	if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
+	    (I915_READ_TAIL(engine) & TAIL_ADDR))
+		idle = false;
+
 	/* No bit for gen2, so assume the CS parser is idle */
 	if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
 		idle = false;
@@ -1100,17 +1272,26 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
  */
 bool intel_engine_is_idle(struct intel_engine_cs *engine)
 {
+	struct drm_i915_private *dev_priv = engine->i915;
+
+	/* More white lies, if wedged, hw state is inconsistent */
+	if (i915_terminally_wedged(&dev_priv->gpu_error))
+		return true;
+
 	/* Any inflight/incomplete requests? */
 	if (!i915_seqno_passed(intel_engine_get_seqno(engine),
 			       intel_engine_last_submit(engine)))
 		return false;
 
+	if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock))
+		return true;
+
 	/* Interrupt/tasklet pending? */
 	if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
 		return false;
 
 	/* Both ports drained, no more ELSP submission? */
-	if (engine->execlist_port[0].request)
+	if (port_request(&engine->execlist_port[0]))
 		return false;
 
 	/* Ring stopped? */
@@ -1151,6 +1332,18 @@ void intel_engines_reset_default_submission(struct drm_i915_private *i915)
 		engine->set_default_submission(engine);
 }
 
+void intel_engines_mark_idle(struct drm_i915_private *i915)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, i915, id) {
+		intel_engine_disarm_breadcrumbs(engine);
+		i915_gem_batch_pool_fini(&engine->batch_pool);
+		engine->no_priolist = false;
+	}
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/mock_engine.c"
 #endif
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index d93c58410bff..860b8c26d29b 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -796,7 +796,7 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 		return false;
 	}
 	if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) &&
-	    cache->plane.rotation != DRM_ROTATE_0) {
+	    cache->plane.rotation != DRM_MODE_ROTATE_0) {
 		fbc->no_fbc_reason = "rotation unsupported";
 		return false;
 	}
@@ -1307,14 +1307,12 @@ static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)
 
 static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)
 {
-#ifdef CONFIG_INTEL_IOMMU
 	/* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
-	if (intel_iommu_gfx_mapped &&
+	if (intel_vtd_active() &&
 	    (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) {
 		DRM_INFO("Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n");
 		return true;
 	}
-#endif
 
 	return false;
 }
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 332254a8eebe..03347c6ae599 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -211,7 +211,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	 * This also validates that any existing fb inherited from the
 	 * BIOS is suitable for own access.
 	 */
-	vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, DRM_ROTATE_0);
+	vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, DRM_MODE_ROTATE_0);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out_unlock;
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index 966e255ca053..d484862cc7df 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -262,7 +262,7 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 		ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
 	else if (IS_GEN7(dev_priv))
 		ivybridge_set_fifo_underrun_reporting(dev, pipe, enable, old);
-	else if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv))
+	else if (INTEL_GEN(dev_priv) >= 8)
 		broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
 
 	return old;
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.c b/drivers/gpu/drm/i915/intel_guc_ct.c
new file mode 100644
index 000000000000..c4cbec140101
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_guc_ct.c
@@ -0,0 +1,461 @@
+/*
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "i915_drv.h"
+#include "intel_guc_ct.h"
+
+enum { CTB_SEND = 0, CTB_RECV = 1 };
+
+enum { CTB_OWNER_HOST = 0 };
+
+void intel_guc_ct_init_early(struct intel_guc_ct *ct)
+{
+	/* we're using static channel owners */
+	ct->host_channel.owner = CTB_OWNER_HOST;
+}
+
+static inline const char *guc_ct_buffer_type_to_str(u32 type)
+{
+	switch (type) {
+	case INTEL_GUC_CT_BUFFER_TYPE_SEND:
+		return "SEND";
+	case INTEL_GUC_CT_BUFFER_TYPE_RECV:
+		return "RECV";
+	default:
+		return "<invalid>";
+	}
+}
+
+static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc,
+				    u32 cmds_addr, u32 size, u32 owner)
+{
+	DRM_DEBUG_DRIVER("CT: desc %p init addr=%#x size=%u owner=%u\n",
+			 desc, cmds_addr, size, owner);
+	memset(desc, 0, sizeof(*desc));
+	desc->addr = cmds_addr;
+	desc->size = size;
+	desc->owner = owner;
+}
+
+static void guc_ct_buffer_desc_reset(struct guc_ct_buffer_desc *desc)
+{
+	DRM_DEBUG_DRIVER("CT: desc %p reset head=%u tail=%u\n",
+			 desc, desc->head, desc->tail);
+	desc->head = 0;
+	desc->tail = 0;
+	desc->is_in_error = 0;
+}
+
+static int guc_action_register_ct_buffer(struct intel_guc *guc,
+					 u32 desc_addr,
+					 u32 type)
+{
+	u32 action[] = {
+		INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER,
+		desc_addr,
+		sizeof(struct guc_ct_buffer_desc),
+		type
+	};
+	int err;
+
+	/* Can't use generic send(), CT registration must go over MMIO */
+	err = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action));
+	if (err)
+		DRM_ERROR("CT: register %s buffer failed; err=%d\n",
+			  guc_ct_buffer_type_to_str(type), err);
+	return err;
+}
+
+static int guc_action_deregister_ct_buffer(struct intel_guc *guc,
+					   u32 owner,
+					   u32 type)
+{
+	u32 action[] = {
+		INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER,
+		owner,
+		type
+	};
+	int err;
+
+	/* Can't use generic send(), CT deregistration must go over MMIO */
+	err = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action));
+	if (err)
+		DRM_ERROR("CT: deregister %s buffer failed; owner=%d err=%d\n",
+			  guc_ct_buffer_type_to_str(type), owner, err);
+	return err;
+}
+
+static bool ctch_is_open(struct intel_guc_ct_channel *ctch)
+{
+	return ctch->vma != NULL;
+}
+
+static int ctch_init(struct intel_guc *guc,
+		     struct intel_guc_ct_channel *ctch)
+{
+	struct i915_vma *vma;
+	void *blob;
+	int err;
+	int i;
+
+	GEM_BUG_ON(ctch->vma);
+
+	/* We allocate 1 page to hold both descriptors and both buffers.
+	 *       ___________.....................
+	 *      |desc (SEND)|                   :
+	 *      |___________|                   PAGE/4
+	 *      :___________....................:
+	 *      |desc (RECV)|                   :
+	 *      |___________|                   PAGE/4
+	 *      :_______________________________:
+	 *      |cmds (SEND)                    |
+	 *      |                               PAGE/4
+	 *      |_______________________________|
+	 *      |cmds (RECV)                    |
+	 *      |                               PAGE/4
+	 *      |_______________________________|
+	 *
+	 * Each message can use a maximum of 32 dwords and we don't expect to
+	 * have more than 1 in flight at any time, so we have enough space.
+	 * Some logic further ahead will rely on the fact that there is only 1
+	 * page and that it is always mapped, so if the size is changed the
+	 * other code will need updating as well.
+	 */
+
+	/* allocate vma */
+	vma = intel_guc_allocate_vma(guc, PAGE_SIZE);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_out;
+	}
+	ctch->vma = vma;
+
+	/* map first page */
+	blob = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+	if (IS_ERR(blob)) {
+		err = PTR_ERR(blob);
+		goto err_vma;
+	}
+	DRM_DEBUG_DRIVER("CT: vma base=%#x\n", guc_ggtt_offset(ctch->vma));
+
+	/* store pointers to desc and cmds */
+	for (i = 0; i < ARRAY_SIZE(ctch->ctbs); i++) {
+		GEM_BUG_ON((i != CTB_SEND) && (i != CTB_RECV));
+		ctch->ctbs[i].desc = blob + PAGE_SIZE/4 * i;
+		ctch->ctbs[i].cmds = blob + PAGE_SIZE/4 * i + PAGE_SIZE/2;
+	}
+
+	return 0;
+
+err_vma:
+	i915_vma_unpin_and_release(&ctch->vma);
+err_out:
+	DRM_DEBUG_DRIVER("CT: channel %d initialization failed; err=%d\n",
+			 ctch->owner, err);
+	return err;
+}
+
+static void ctch_fini(struct intel_guc *guc,
+		      struct intel_guc_ct_channel *ctch)
+{
+	GEM_BUG_ON(!ctch->vma);
+
+	i915_gem_object_unpin_map(ctch->vma->obj);
+	i915_vma_unpin_and_release(&ctch->vma);
+}
+
+static int ctch_open(struct intel_guc *guc,
+		     struct intel_guc_ct_channel *ctch)
+{
+	u32 base;
+	int err;
+	int i;
+
+	DRM_DEBUG_DRIVER("CT: channel %d reopen=%s\n",
+			 ctch->owner, yesno(ctch_is_open(ctch)));
+
+	if (!ctch->vma) {
+		err = ctch_init(guc, ctch);
+		if (unlikely(err))
+			goto err_out;
+	}
+
+	/* vma should be already allocated and map'ed */
+	base = guc_ggtt_offset(ctch->vma);
+
+	/* (re)initialize descriptors
+	 * cmds buffers are in the second half of the blob page
+	 */
+	for (i = 0; i < ARRAY_SIZE(ctch->ctbs); i++) {
+		GEM_BUG_ON((i != CTB_SEND) && (i != CTB_RECV));
+		guc_ct_buffer_desc_init(ctch->ctbs[i].desc,
+					base + PAGE_SIZE/4 * i + PAGE_SIZE/2,
+					PAGE_SIZE/4,
+					ctch->owner);
+	}
+
+	/* register buffers, starting wirh RECV buffer
+	 * descriptors are in first half of the blob
+	 */
+	err = guc_action_register_ct_buffer(guc,
+					    base + PAGE_SIZE/4 * CTB_RECV,
+					    INTEL_GUC_CT_BUFFER_TYPE_RECV);
+	if (unlikely(err))
+		goto err_fini;
+
+	err = guc_action_register_ct_buffer(guc,
+					    base + PAGE_SIZE/4 * CTB_SEND,
+					    INTEL_GUC_CT_BUFFER_TYPE_SEND);
+	if (unlikely(err))
+		goto err_deregister;
+
+	return 0;
+
+err_deregister:
+	guc_action_deregister_ct_buffer(guc,
+					ctch->owner,
+					INTEL_GUC_CT_BUFFER_TYPE_RECV);
+err_fini:
+	ctch_fini(guc, ctch);
+err_out:
+	DRM_ERROR("CT: can't open channel %d; err=%d\n", ctch->owner, err);
+	return err;
+}
+
+static void ctch_close(struct intel_guc *guc,
+		       struct intel_guc_ct_channel *ctch)
+{
+	GEM_BUG_ON(!ctch_is_open(ctch));
+
+	guc_action_deregister_ct_buffer(guc,
+					ctch->owner,
+					INTEL_GUC_CT_BUFFER_TYPE_SEND);
+	guc_action_deregister_ct_buffer(guc,
+					ctch->owner,
+					INTEL_GUC_CT_BUFFER_TYPE_RECV);
+	ctch_fini(guc, ctch);
+}
+
+static u32 ctch_get_next_fence(struct intel_guc_ct_channel *ctch)
+{
+	/* For now it's trivial */
+	return ++ctch->next_fence;
+}
+
+static int ctb_write(struct intel_guc_ct_buffer *ctb,
+		     const u32 *action,
+		     u32 len /* in dwords */,
+		     u32 fence)
+{
+	struct guc_ct_buffer_desc *desc = ctb->desc;
+	u32 head = desc->head / 4;	/* in dwords */
+	u32 tail = desc->tail / 4;	/* in dwords */
+	u32 size = desc->size / 4;	/* in dwords */
+	u32 used;			/* in dwords */
+	u32 header;
+	u32 *cmds = ctb->cmds;
+	unsigned int i;
+
+	GEM_BUG_ON(desc->size % 4);
+	GEM_BUG_ON(desc->head % 4);
+	GEM_BUG_ON(desc->tail % 4);
+	GEM_BUG_ON(tail >= size);
+
+	/*
+	 * tail == head condition indicates empty. GuC FW does not support
+	 * using up the entire buffer to get tail == head meaning full.
+	 */
+	if (tail < head)
+		used = (size - head) + tail;
+	else
+		used = tail - head;
+
+	/* make sure there is a space including extra dw for the fence */
+	if (unlikely(used + len + 1 >= size))
+		return -ENOSPC;
+
+	/* Write the message. The format is the following:
+	 * DW0: header (including action code)
+	 * DW1: fence
+	 * DW2+: action data
+	 */
+	header = (len << GUC_CT_MSG_LEN_SHIFT) |
+		 (GUC_CT_MSG_WRITE_FENCE_TO_DESC) |
+		 (action[0] << GUC_CT_MSG_ACTION_SHIFT);
+
+	cmds[tail] = header;
+	tail = (tail + 1) % size;
+
+	cmds[tail] = fence;
+	tail = (tail + 1) % size;
+
+	for (i = 1; i < len; i++) {
+		cmds[tail] = action[i];
+		tail = (tail + 1) % size;
+	}
+
+	/* now update desc tail (back in bytes) */
+	desc->tail = tail * 4;
+	GEM_BUG_ON(desc->tail > desc->size);
+
+	return 0;
+}
+
+/* Wait for the response from the GuC.
+ * @fence:	response fence
+ * @status:	placeholder for status
+ * return:	0 response received (status is valid)
+ *		-ETIMEDOUT no response within hardcoded timeout
+ *		-EPROTO no response, ct buffer was in error
+ */
+static int wait_for_response(struct guc_ct_buffer_desc *desc,
+			     u32 fence,
+			     u32 *status)
+{
+	int err;
+
+	/*
+	 * Fast commands should complete in less than 10us, so sample quickly
+	 * up to that length of time, then switch to a slower sleep-wait loop.
+	 * No GuC command should ever take longer than 10ms.
+	 */
+#define done (READ_ONCE(desc->fence) == fence)
+	err = wait_for_us(done, 10);
+	if (err)
+		err = wait_for(done, 10);
+#undef done
+
+	if (unlikely(err)) {
+		DRM_ERROR("CT: fence %u failed; reported fence=%u\n",
+			  fence, desc->fence);
+
+		if (WARN_ON(desc->is_in_error)) {
+			/* Something went wrong with the messaging, try to reset
+			 * the buffer and hope for the best
+			 */
+			guc_ct_buffer_desc_reset(desc);
+			err = -EPROTO;
+		}
+	}
+
+	*status = desc->status;
+	return err;
+}
+
+static int ctch_send(struct intel_guc *guc,
+		     struct intel_guc_ct_channel *ctch,
+		     const u32 *action,
+		     u32 len,
+		     u32 *status)
+{
+	struct intel_guc_ct_buffer *ctb = &ctch->ctbs[CTB_SEND];
+	struct guc_ct_buffer_desc *desc = ctb->desc;
+	u32 fence;
+	int err;
+
+	GEM_BUG_ON(!ctch_is_open(ctch));
+	GEM_BUG_ON(!len);
+	GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK);
+
+	fence = ctch_get_next_fence(ctch);
+	err = ctb_write(ctb, action, len, fence);
+	if (unlikely(err))
+		return err;
+
+	intel_guc_notify(guc);
+
+	err = wait_for_response(desc, fence, status);
+	if (unlikely(err))
+		return err;
+	if (*status != INTEL_GUC_STATUS_SUCCESS)
+		return -EIO;
+	return 0;
+}
+
+/*
+ * Command Transport (CT) buffer based GuC send function.
+ */
+static int intel_guc_send_ct(struct intel_guc *guc, const u32 *action, u32 len)
+{
+	struct intel_guc_ct_channel *ctch = &guc->ct.host_channel;
+	u32 status = ~0; /* undefined */
+	int err;
+
+	mutex_lock(&guc->send_mutex);
+
+	err = ctch_send(guc, ctch, action, len, &status);
+	if (unlikely(err)) {
+		DRM_ERROR("CT: send action %#X failed; err=%d status=%#X\n",
+			  action[0], err, status);
+	}
+
+	mutex_unlock(&guc->send_mutex);
+	return err;
+}
+
+/**
+ * Enable buffer based command transport
+ * Shall only be called for platforms with HAS_GUC_CT.
+ * @guc:	the guc
+ * return:	0 on success
+ *		non-zero on failure
+ */
+int intel_guc_enable_ct(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_guc_ct_channel *ctch = &guc->ct.host_channel;
+	int err;
+
+	GEM_BUG_ON(!HAS_GUC_CT(dev_priv));
+
+	err = ctch_open(guc, ctch);
+	if (unlikely(err))
+		return err;
+
+	/* Switch into cmd transport buffer based send() */
+	guc->send = intel_guc_send_ct;
+	DRM_INFO("CT: %s\n", enableddisabled(true));
+	return 0;
+}
+
+/**
+ * Disable buffer based command transport.
+ * Shall only be called for platforms with HAS_GUC_CT.
+ * @guc: the guc
+ */
+void intel_guc_disable_ct(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_guc_ct_channel *ctch = &guc->ct.host_channel;
+
+	GEM_BUG_ON(!HAS_GUC_CT(dev_priv));
+
+	if (!ctch_is_open(ctch))
+		return;
+
+	ctch_close(guc, ctch);
+
+	/* Disable send */
+	guc->send = intel_guc_send_nop;
+	DRM_INFO("CT: %s\n", enableddisabled(false));
+}
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.h b/drivers/gpu/drm/i915/intel_guc_ct.h
new file mode 100644
index 000000000000..6d97f36fcc62
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_guc_ct.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef _INTEL_GUC_CT_H_
+#define _INTEL_GUC_CT_H_
+
+struct intel_guc;
+struct i915_vma;
+
+#include "intel_guc_fwif.h"
+
+/**
+ * DOC: Command Transport (CT).
+ *
+ * Buffer based command transport is a replacement for MMIO based mechanism.
+ * It can be used to perform both host-2-guc and guc-to-host communication.
+ */
+
+/** Represents single command transport buffer.
+ *
+ * A single command transport buffer consists of two parts, the header
+ * record (command transport buffer descriptor) and the actual buffer which
+ * holds the commands.
+ *
+ * @desc: pointer to the buffer descriptor
+ * @cmds: pointer to the commands buffer
+ */
+struct intel_guc_ct_buffer {
+	struct guc_ct_buffer_desc *desc;
+	u32 *cmds;
+};
+
+/** Represents pair of command transport buffers.
+ *
+ * Buffers go in pairs to allow bi-directional communication.
+ * To simplify the code we place both of them in the same vma.
+ * Buffers from the same pair must share unique owner id.
+ *
+ * @vma: pointer to the vma with pair of CT buffers
+ * @ctbs: buffers for sending(0) and receiving(1) commands
+ * @owner: unique identifier
+ * @next_fence: fence to be used with next send command
+ */
+struct intel_guc_ct_channel {
+	struct i915_vma *vma;
+	struct intel_guc_ct_buffer ctbs[2];
+	u32 owner;
+	u32 next_fence;
+};
+
+/** Holds all command transport channels.
+ *
+ * @host_channel: main channel used by the host
+ */
+struct intel_guc_ct {
+	struct intel_guc_ct_channel host_channel;
+	/* other channels are tbd */
+};
+
+void intel_guc_ct_init_early(struct intel_guc_ct *ct);
+
+/* XXX: move to intel_uc.h ? don't fit there either */
+int intel_guc_enable_ct(struct intel_guc *guc);
+void intel_guc_disable_ct(struct intel_guc *guc);
+
+#endif /* _INTEL_GUC_CT_H_ */
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index cb36cbf3818f..5fa286074811 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -23,8 +23,8 @@
 #ifndef _INTEL_GUC_FWIF_H
 #define _INTEL_GUC_FWIF_H
 
-#define GFXCORE_FAMILY_GEN9		12
-#define GFXCORE_FAMILY_UNKNOWN		0x7fffffff
+#define GUC_CORE_FAMILY_GEN9		12
+#define GUC_CORE_FAMILY_UNKNOWN		0x7fffffff
 
 #define GUC_CLIENT_PRIORITY_KMD_HIGH	0
 #define GUC_CLIENT_PRIORITY_HIGH	1
@@ -331,6 +331,47 @@ struct guc_stage_desc {
 	u64 desc_private;
 } __packed;
 
+/*
+ * Describes single command transport buffer.
+ * Used by both guc-master and clients.
+ */
+struct guc_ct_buffer_desc {
+	u32 addr;		/* gfx address */
+	u64 host_private;	/* host private data */
+	u32 size;		/* size in bytes */
+	u32 head;		/* offset updated by GuC*/
+	u32 tail;		/* offset updated by owner */
+	u32 is_in_error;	/* error indicator */
+	u32 fence;		/* fence updated by GuC */
+	u32 status;		/* status updated by GuC */
+	u32 owner;		/* id of the channel owner */
+	u32 owner_sub_id;	/* owner-defined field for extra tracking */
+	u32 reserved[5];
+} __packed;
+
+/* Type of command transport buffer */
+#define INTEL_GUC_CT_BUFFER_TYPE_SEND	0x0u
+#define INTEL_GUC_CT_BUFFER_TYPE_RECV	0x1u
+
+/*
+ * Definition of the command transport message header (DW0)
+ *
+ * bit[4..0]	message len (in dwords)
+ * bit[7..5]	reserved
+ * bit[8]	write fence to desc
+ * bit[9]	write status to H2G buff
+ * bit[10]	send status (via G2H)
+ * bit[15..11]	reserved
+ * bit[31..16]	action code
+ */
+#define GUC_CT_MSG_LEN_SHIFT			0
+#define GUC_CT_MSG_LEN_MASK			0x1F
+#define GUC_CT_MSG_WRITE_FENCE_TO_DESC		(1 << 8)
+#define GUC_CT_MSG_WRITE_STATUS_TO_BUFF		(1 << 9)
+#define GUC_CT_MSG_SEND_STATUS			(1 << 10)
+#define GUC_CT_MSG_ACTION_SHIFT			16
+#define GUC_CT_MSG_ACTION_MASK			0xFFFF
+
 #define GUC_FORCEWAKE_RENDER	(1 << 0)
 #define GUC_FORCEWAKE_MEDIA	(1 << 1)
 
@@ -515,6 +556,8 @@ enum intel_guc_action {
 	INTEL_GUC_ACTION_EXIT_S_STATE = 0x502,
 	INTEL_GUC_ACTION_SLPC_REQUEST = 0x3003,
 	INTEL_GUC_ACTION_AUTHENTICATE_HUC = 0x4000,
+	INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER = 0x4505,
+	INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER = 0x4506,
 	INTEL_GUC_ACTION_UK_LOG_ENABLE_LOGGING = 0x0E000,
 	INTEL_GUC_ACTION_LIMIT
 };
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 8a1a023e48b2..8b0ae7fce7f2 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -61,6 +61,9 @@
 #define KBL_FW_MAJOR 9
 #define KBL_FW_MINOR 14
 
+#define GLK_FW_MAJOR 10
+#define GLK_FW_MINOR 56
+
 #define GUC_FW_PATH(platform, major, minor) \
        "i915/" __stringify(platform) "_guc_ver" __stringify(major) "_" __stringify(minor) ".bin"
 
@@ -73,6 +76,8 @@ MODULE_FIRMWARE(I915_BXT_GUC_UCODE);
 #define I915_KBL_GUC_UCODE GUC_FW_PATH(kbl, KBL_FW_MAJOR, KBL_FW_MINOR)
 MODULE_FIRMWARE(I915_KBL_GUC_UCODE);
 
+#define I915_GLK_GUC_UCODE GUC_FW_PATH(glk, GLK_FW_MAJOR, GLK_FW_MINOR)
+
 
 static u32 get_gttype(struct drm_i915_private *dev_priv)
 {
@@ -86,11 +91,11 @@ static u32 get_core_family(struct drm_i915_private *dev_priv)
 
 	switch (gen) {
 	case 9:
-		return GFXCORE_FAMILY_GEN9;
+		return GUC_CORE_FAMILY_GEN9;
 
 	default:
-		WARN(1, "GEN%d does not support GuC operation!\n", gen);
-		return GFXCORE_FAMILY_UNKNOWN;
+		MISSING_CASE(gen);
+		return GUC_CORE_FAMILY_UNKNOWN;
 	}
 }
 
@@ -280,10 +285,6 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	/* init WOPCM */
-	I915_WRITE(GUC_WOPCM_SIZE, intel_guc_wopcm_size(dev_priv));
-	I915_WRITE(DMA_GUC_WOPCM_OFFSET, GUC_WOPCM_OFFSET_VALUE);
-
 	/* Enable MIA caching. GuC clock gating is disabled. */
 	I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
 
@@ -401,10 +402,14 @@ int intel_guc_select_fw(struct intel_guc *guc)
 		guc->fw.path = I915_BXT_GUC_UCODE;
 		guc->fw.major_ver_wanted = BXT_FW_MAJOR;
 		guc->fw.minor_ver_wanted = BXT_FW_MINOR;
-	} else if (IS_KABYLAKE(dev_priv)) {
+	} else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
 		guc->fw.path = I915_KBL_GUC_UCODE;
 		guc->fw.major_ver_wanted = KBL_FW_MAJOR;
 		guc->fw.minor_ver_wanted = KBL_FW_MINOR;
+	} else if (IS_GEMINILAKE(dev_priv)) {
+		guc->fw.path = I915_GLK_GUC_UCODE;
+		guc->fw.major_ver_wanted = GLK_FW_MAJOR;
+		guc->fw.minor_ver_wanted = GLK_FW_MINOR;
 	} else {
 		DRM_ERROR("No GuC firmware known for platform with GuC!\n");
 		return -ENOENT;
diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c
index 6fb63a3c65b0..16d3b8719cab 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/intel_guc_log.c
@@ -359,12 +359,16 @@ static int guc_log_runtime_create(struct intel_guc *guc)
 	void *vaddr;
 	struct rchan *guc_log_relay_chan;
 	size_t n_subbufs, subbuf_size;
-	int ret = 0;
+	int ret;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
 	GEM_BUG_ON(guc_log_has_runtime(guc));
 
+	ret = i915_gem_object_set_to_wc_domain(guc->log.vma->obj, true);
+	if (ret)
+		return ret;
+
 	/* Create a WC (Uncached for read) vmalloc mapping of log
 	 * buffer pages, so that we can directly get the data
 	 * (up-to-date) from memory.
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index e1ab6432a914..52d5b82790d9 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -51,6 +51,32 @@ static bool is_supported_device(struct drm_i915_private *dev_priv)
 }
 
 /**
+ * intel_gvt_sanitize_options - sanitize GVT related options
+ * @dev_priv: drm i915 private data
+ *
+ * This function is called at the i915 options sanitize stage.
+ */
+void intel_gvt_sanitize_options(struct drm_i915_private *dev_priv)
+{
+	if (!i915.enable_gvt)
+		return;
+
+	if (intel_vgpu_active(dev_priv)) {
+		DRM_INFO("GVT-g is disabled for guest\n");
+		goto bail;
+	}
+
+	if (!is_supported_device(dev_priv)) {
+		DRM_INFO("Unsupported device. GVT-g is disabled\n");
+		goto bail;
+	}
+
+	return;
+bail:
+	i915.enable_gvt = 0;
+}
+
+/**
  * intel_gvt_init - initialize GVT components
  * @dev_priv: drm i915 private data
  *
@@ -69,19 +95,14 @@ int intel_gvt_init(struct drm_i915_private *dev_priv)
 		return 0;
 	}
 
-	if (intel_vgpu_active(dev_priv)) {
-		DRM_DEBUG_DRIVER("GVT-g is disabled for guest\n");
-		goto bail;
-	}
-
-	if (!is_supported_device(dev_priv)) {
-		DRM_DEBUG_DRIVER("Unsupported device. GVT-g is disabled\n");
-		goto bail;
+	if (!i915.enable_execlists) {
+		DRM_ERROR("i915 GVT-g loading failed due to disabled execlists mode\n");
+		return -EIO;
 	}
 
-	if (!i915.enable_execlists) {
-		DRM_INFO("GPU guest virtualisation [GVT-g] disabled due to disabled execlist submission [i915.enable_execlists module parameter]\n");
-		goto bail;
+	if (i915.enable_guc_submission) {
+		DRM_ERROR("i915 GVT-g loading failed due to Graphics virtualization is not yet supported with GuC submission\n");
+		return -EIO;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/i915/intel_gvt.h b/drivers/gpu/drm/i915/intel_gvt.h
index 25df2d65b985..61b246470282 100644
--- a/drivers/gpu/drm/i915/intel_gvt.h
+++ b/drivers/gpu/drm/i915/intel_gvt.h
@@ -32,6 +32,7 @@ void intel_gvt_cleanup(struct drm_i915_private *dev_priv);
 int intel_gvt_init_device(struct drm_i915_private *dev_priv);
 void intel_gvt_clean_device(struct drm_i915_private *dev_priv);
 int intel_gvt_init_host(void);
+void intel_gvt_sanitize_options(struct drm_i915_private *dev_priv);
 #else
 static inline int intel_gvt_init(struct drm_i915_private *dev_priv)
 {
@@ -40,6 +41,10 @@ static inline int intel_gvt_init(struct drm_i915_private *dev_priv)
 static inline void intel_gvt_cleanup(struct drm_i915_private *dev_priv)
 {
 }
+
+static inline void intel_gvt_sanitize_options(struct drm_i915_private *dev_priv)
+{
+}
 #endif
 
 #endif /* _INTEL_GVT_H_ */
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
index dce742243ba6..9b0ece427bdc 100644
--- a/drivers/gpu/drm/i915/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -407,7 +407,7 @@ static void hangcheck_declare_hang(struct drm_i915_private *i915,
 				 "%s, ", engine->name);
 	msg[len-2] = '\0';
 
-	return i915_handle_error(i915, hung, msg);
+	return i915_handle_error(i915, hung, "%s", msg);
 }
 
 /*
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 1d623b5e09d6..ec0779a52d53 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -1218,7 +1218,8 @@ static int intel_hdmi_source_max_tmds_clock(struct drm_i915_private *dev_priv)
 }
 
 static int hdmi_port_clock_limit(struct intel_hdmi *hdmi,
-				 bool respect_downstream_limits)
+				 bool respect_downstream_limits,
+				 bool force_dvi)
 {
 	struct drm_device *dev = intel_hdmi_to_dev(hdmi);
 	int max_tmds_clock = intel_hdmi_source_max_tmds_clock(to_i915(dev));
@@ -1234,7 +1235,7 @@ static int hdmi_port_clock_limit(struct intel_hdmi *hdmi,
 		if (info->max_tmds_clock)
 			max_tmds_clock = min(max_tmds_clock,
 					     info->max_tmds_clock);
-		else if (!hdmi->has_hdmi_sink)
+		else if (!hdmi->has_hdmi_sink || force_dvi)
 			max_tmds_clock = min(max_tmds_clock, 165000);
 	}
 
@@ -1243,13 +1244,14 @@ static int hdmi_port_clock_limit(struct intel_hdmi *hdmi,
 
 static enum drm_mode_status
 hdmi_port_clock_valid(struct intel_hdmi *hdmi,
-		      int clock, bool respect_downstream_limits)
+		      int clock, bool respect_downstream_limits,
+		      bool force_dvi)
 {
 	struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi));
 
 	if (clock < 25000)
 		return MODE_CLOCK_LOW;
-	if (clock > hdmi_port_clock_limit(hdmi, respect_downstream_limits))
+	if (clock > hdmi_port_clock_limit(hdmi, respect_downstream_limits, force_dvi))
 		return MODE_CLOCK_HIGH;
 
 	/* BXT DPLL can't generate 223-240 MHz */
@@ -1273,6 +1275,8 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 	enum drm_mode_status status;
 	int clock;
 	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+	bool force_dvi =
+		READ_ONCE(to_intel_digital_connector_state(connector->state)->force_audio) == HDMI_AUDIO_OFF_DVI;
 
 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return MODE_NO_DBLESCAN;
@@ -1289,11 +1293,11 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 		clock *= 2;
 
 	/* check if we can do 8bpc */
-	status = hdmi_port_clock_valid(hdmi, clock, true);
+	status = hdmi_port_clock_valid(hdmi, clock, true, force_dvi);
 
 	/* if we can't do 8bpc we may still be able to do 12bpc */
-	if (!HAS_GMCH_DISPLAY(dev_priv) && status != MODE_OK)
-		status = hdmi_port_clock_valid(hdmi, clock * 3 / 2, true);
+	if (!HAS_GMCH_DISPLAY(dev_priv) && status != MODE_OK && hdmi->has_hdmi_sink && !force_dvi)
+		status = hdmi_port_clock_valid(hdmi, clock * 3 / 2, true, force_dvi);
 
 	return status;
 }
@@ -1327,6 +1331,11 @@ static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 			return false;
 	}
 
+	/* Display Wa #1139 */
+	if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1) &&
+	    crtc_state->base.adjusted_mode.htotal > 5460)
+		return false;
+
 	return true;
 }
 
@@ -1338,16 +1347,19 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	struct drm_scdc *scdc = &conn_state->connector->display_info.hdmi.scdc;
+	struct intel_digital_connector_state *intel_conn_state =
+		to_intel_digital_connector_state(conn_state);
 	int clock_8bpc = pipe_config->base.adjusted_mode.crtc_clock;
 	int clock_12bpc = clock_8bpc * 3 / 2;
 	int desired_bpp;
+	bool force_dvi = intel_conn_state->force_audio == HDMI_AUDIO_OFF_DVI;
 
-	pipe_config->has_hdmi_sink = intel_hdmi->has_hdmi_sink;
+	pipe_config->has_hdmi_sink = !force_dvi && intel_hdmi->has_hdmi_sink;
 
 	if (pipe_config->has_hdmi_sink)
 		pipe_config->has_infoframe = true;
 
-	if (intel_hdmi->color_range_auto) {
+	if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) {
 		/* See CEA-861-E - 5.1 Default Encoding Parameters */
 		pipe_config->limited_color_range =
 			pipe_config->has_hdmi_sink &&
@@ -1355,7 +1367,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 			HDMI_QUANTIZATION_RANGE_LIMITED;
 	} else {
 		pipe_config->limited_color_range =
-			intel_hdmi->limited_color_range;
+			intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_LIMITED;
 	}
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) {
@@ -1367,8 +1379,13 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv))
 		pipe_config->has_pch_encoder = true;
 
-	if (pipe_config->has_hdmi_sink && intel_hdmi->has_audio)
-		pipe_config->has_audio = true;
+	if (pipe_config->has_hdmi_sink) {
+		if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO)
+			pipe_config->has_audio = intel_hdmi->has_audio;
+		else
+			pipe_config->has_audio =
+				intel_conn_state->force_audio == HDMI_AUDIO_ON;
+	}
 
 	/*
 	 * HDMI is either 12 or 8, so if the display lets 10bpc sneak
@@ -1376,8 +1393,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 	 * outputs. We also need to check that the higher clock still fits
 	 * within limits.
 	 */
-	if (pipe_config->pipe_bpp > 8*3 && pipe_config->has_hdmi_sink &&
-	    hdmi_port_clock_valid(intel_hdmi, clock_12bpc, true) == MODE_OK &&
+	if (pipe_config->pipe_bpp > 8*3 && pipe_config->has_hdmi_sink && !force_dvi &&
+	    hdmi_port_clock_valid(intel_hdmi, clock_12bpc, true, force_dvi) == MODE_OK &&
 	    hdmi_12bpc_possible(pipe_config)) {
 		DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n");
 		desired_bpp = 12*3;
@@ -1392,18 +1409,18 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 	}
 
 	if (!pipe_config->bw_constrained) {
-		DRM_DEBUG_KMS("forcing pipe bpc to %i for HDMI\n", desired_bpp);
+		DRM_DEBUG_KMS("forcing pipe bpp to %i for HDMI\n", desired_bpp);
 		pipe_config->pipe_bpp = desired_bpp;
 	}
 
 	if (hdmi_port_clock_valid(intel_hdmi, pipe_config->port_clock,
-				  false) != MODE_OK) {
+				  false, force_dvi) != MODE_OK) {
 		DRM_DEBUG_KMS("unsupported HDMI clock, rejecting mode\n");
 		return false;
 	}
 
 	/* Set user selected PAR to incoming mode's member */
-	adjusted_mode->picture_aspect_ratio = intel_hdmi->aspect_ratio;
+	adjusted_mode->picture_aspect_ratio = conn_state->picture_aspect_ratio;
 
 	pipe_config->lane_count = 4;
 
@@ -1504,13 +1521,7 @@ intel_hdmi_set_edid(struct drm_connector *connector)
 			drm_rgb_quant_range_selectable(edid);
 
 		intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
-		if (intel_hdmi->force_audio != HDMI_AUDIO_AUTO)
-			intel_hdmi->has_audio =
-				intel_hdmi->force_audio == HDMI_AUDIO_ON;
-
-		if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI)
-			intel_hdmi->has_hdmi_sink =
-				drm_detect_hdmi_monitor(edid);
+		intel_hdmi->has_hdmi_sink = drm_detect_hdmi_monitor(edid);
 
 		connected = true;
 	}
@@ -1572,108 +1583,6 @@ static int intel_hdmi_get_modes(struct drm_connector *connector)
 	return intel_connector_update_modes(connector, edid);
 }
 
-static bool
-intel_hdmi_detect_audio(struct drm_connector *connector)
-{
-	bool has_audio = false;
-	struct edid *edid;
-
-	edid = to_intel_connector(connector)->detect_edid;
-	if (edid && edid->input & DRM_EDID_INPUT_DIGITAL)
-		has_audio = drm_detect_monitor_audio(edid);
-
-	return has_audio;
-}
-
-static int
-intel_hdmi_set_property(struct drm_connector *connector,
-			struct drm_property *property,
-			uint64_t val)
-{
-	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-	struct intel_digital_port *intel_dig_port =
-		hdmi_to_dig_port(intel_hdmi);
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	int ret;
-
-	ret = drm_object_property_set_value(&connector->base, property, val);
-	if (ret)
-		return ret;
-
-	if (property == dev_priv->force_audio_property) {
-		enum hdmi_force_audio i = val;
-		bool has_audio;
-
-		if (i == intel_hdmi->force_audio)
-			return 0;
-
-		intel_hdmi->force_audio = i;
-
-		if (i == HDMI_AUDIO_AUTO)
-			has_audio = intel_hdmi_detect_audio(connector);
-		else
-			has_audio = (i == HDMI_AUDIO_ON);
-
-		if (i == HDMI_AUDIO_OFF_DVI)
-			intel_hdmi->has_hdmi_sink = 0;
-
-		intel_hdmi->has_audio = has_audio;
-		goto done;
-	}
-
-	if (property == dev_priv->broadcast_rgb_property) {
-		bool old_auto = intel_hdmi->color_range_auto;
-		bool old_range = intel_hdmi->limited_color_range;
-
-		switch (val) {
-		case INTEL_BROADCAST_RGB_AUTO:
-			intel_hdmi->color_range_auto = true;
-			break;
-		case INTEL_BROADCAST_RGB_FULL:
-			intel_hdmi->color_range_auto = false;
-			intel_hdmi->limited_color_range = false;
-			break;
-		case INTEL_BROADCAST_RGB_LIMITED:
-			intel_hdmi->color_range_auto = false;
-			intel_hdmi->limited_color_range = true;
-			break;
-		default:
-			return -EINVAL;
-		}
-
-		if (old_auto == intel_hdmi->color_range_auto &&
-		    old_range == intel_hdmi->limited_color_range)
-			return 0;
-
-		goto done;
-	}
-
-	if (property == connector->dev->mode_config.aspect_ratio_property) {
-		switch (val) {
-		case DRM_MODE_PICTURE_ASPECT_NONE:
-			intel_hdmi->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
-			break;
-		case DRM_MODE_PICTURE_ASPECT_4_3:
-			intel_hdmi->aspect_ratio = HDMI_PICTURE_ASPECT_4_3;
-			break;
-		case DRM_MODE_PICTURE_ASPECT_16_9:
-			intel_hdmi->aspect_ratio = HDMI_PICTURE_ASPECT_16_9;
-			break;
-		default:
-			return -EINVAL;
-		}
-		goto done;
-	}
-
-	return -EINVAL;
-
-done:
-	if (intel_dig_port->base.base.crtc)
-		intel_crtc_restore_mode(intel_dig_port->base.base.crtc);
-
-	return 0;
-}
-
 static void intel_hdmi_pre_enable(struct intel_encoder *encoder,
 				  struct intel_crtc_state *pipe_config,
 				  struct drm_connector_state *conn_state)
@@ -1798,18 +1707,20 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
 	.detect = intel_hdmi_detect,
 	.force = intel_hdmi_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = intel_hdmi_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.atomic_get_property = intel_digital_connector_atomic_get_property,
+	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_hdmi_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
 };
 
 static const struct drm_connector_helper_funcs intel_hdmi_connector_helper_funcs = {
 	.get_modes = intel_hdmi_get_modes,
 	.mode_valid = intel_hdmi_mode_valid,
+	.atomic_check = intel_digital_connector_atomic_check,
 };
 
 static const struct drm_encoder_funcs intel_hdmi_enc_funcs = {
@@ -1821,9 +1732,8 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *c
 {
 	intel_attach_force_audio_property(connector);
 	intel_attach_broadcast_rgb_property(connector);
-	intel_hdmi->color_range_auto = true;
 	intel_attach_aspect_ratio_property(connector);
-	intel_hdmi->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
+	connector->state->picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
 }
 
 /*
@@ -1892,19 +1802,21 @@ static u8 intel_hdmi_ddc_pin(struct drm_i915_private *dev_priv,
 
 	switch (port) {
 	case PORT_B:
-		if (IS_GEN9_LP(dev_priv))
+		if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv))
 			ddc_pin = GMBUS_PIN_1_BXT;
 		else
 			ddc_pin = GMBUS_PIN_DPB;
 		break;
 	case PORT_C:
-		if (IS_GEN9_LP(dev_priv))
+		if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv))
 			ddc_pin = GMBUS_PIN_2_BXT;
 		else
 			ddc_pin = GMBUS_PIN_DPC;
 		break;
 	case PORT_D:
-		if (IS_CHERRYVIEW(dev_priv))
+		if (HAS_PCH_CNP(dev_priv))
+			ddc_pin = GMBUS_PIN_4_CNP;
+		else if (IS_CHERRYVIEW(dev_priv))
 			ddc_pin = GMBUS_PIN_DPD_CHV;
 		else
 			ddc_pin = GMBUS_PIN_DPD;
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index 9ee819666a4c..6145fa0d6773 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -52,6 +52,10 @@
 #define KBL_HUC_FW_MINOR 00
 #define KBL_BLD_NUM 1810
 
+#define GLK_HUC_FW_MAJOR 02
+#define GLK_HUC_FW_MINOR 00
+#define GLK_BLD_NUM 1748
+
 #define HUC_FW_PATH(platform, major, minor, bld_num) \
 	"i915/" __stringify(platform) "_huc_ver" __stringify(major) "_" \
 	__stringify(minor) "_" __stringify(bld_num) ".bin"
@@ -68,6 +72,9 @@ MODULE_FIRMWARE(I915_BXT_HUC_UCODE);
 	KBL_HUC_FW_MINOR, KBL_BLD_NUM)
 MODULE_FIRMWARE(I915_KBL_HUC_UCODE);
 
+#define I915_GLK_HUC_UCODE HUC_FW_PATH(glk, GLK_HUC_FW_MAJOR, \
+	GLK_HUC_FW_MINOR, GLK_BLD_NUM)
+
 /**
  * huc_ucode_xfer() - DMA's the firmware
  * @dev_priv: the drm_i915_private device
@@ -99,11 +106,6 @@ static int huc_ucode_xfer(struct drm_i915_private *dev_priv)
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	/* init WOPCM */
-	I915_WRITE(GUC_WOPCM_SIZE, intel_guc_wopcm_size(dev_priv));
-	I915_WRITE(DMA_GUC_WOPCM_OFFSET, GUC_WOPCM_OFFSET_VALUE |
-			HUC_LOADING_AGENT_GUC);
-
 	/* Set the source address for the uCode */
 	offset = guc_ggtt_offset(vma) + huc_fw->header_offset;
 	I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
@@ -165,10 +167,14 @@ void intel_huc_select_fw(struct intel_huc *huc)
 		huc->fw.path = I915_BXT_HUC_UCODE;
 		huc->fw.major_ver_wanted = BXT_HUC_FW_MAJOR;
 		huc->fw.minor_ver_wanted = BXT_HUC_FW_MINOR;
-	} else if (IS_KABYLAKE(dev_priv)) {
+	} else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
 		huc->fw.path = I915_KBL_HUC_UCODE;
 		huc->fw.major_ver_wanted = KBL_HUC_FW_MAJOR;
 		huc->fw.minor_ver_wanted = KBL_HUC_FW_MINOR;
+	} else if (IS_GEMINILAKE(dev_priv)) {
+		huc->fw.path = I915_GLK_HUC_UCODE;
+		huc->fw.major_ver_wanted = GLK_HUC_FW_MAJOR;
+		huc->fw.minor_ver_wanted = GLK_HUC_FW_MINOR;
 	} else {
 		DRM_ERROR("No HuC firmware known for platform with HuC!\n");
 		return;
@@ -186,68 +192,36 @@ void intel_huc_select_fw(struct intel_huc *huc)
  * earlier call to intel_huc_init(), so here we need only check that
  * is succeeded, and then transfer the image to the h/w.
  *
- * Return:	non-zero code on error
  */
-int intel_huc_init_hw(struct intel_huc *huc)
+void intel_huc_init_hw(struct intel_huc *huc)
 {
 	struct drm_i915_private *dev_priv = huc_to_i915(huc);
 	int err;
 
-	if (huc->fw.fetch_status == INTEL_UC_FIRMWARE_NONE)
-		return 0;
-
 	DRM_DEBUG_DRIVER("%s fw status: fetch %s, load %s\n",
 		huc->fw.path,
 		intel_uc_fw_status_repr(huc->fw.fetch_status),
 		intel_uc_fw_status_repr(huc->fw.load_status));
 
-	if (huc->fw.fetch_status == INTEL_UC_FIRMWARE_SUCCESS &&
-	    huc->fw.load_status == INTEL_UC_FIRMWARE_FAIL)
-		return -ENOEXEC;
+	if (huc->fw.fetch_status != INTEL_UC_FIRMWARE_SUCCESS)
+		return;
 
 	huc->fw.load_status = INTEL_UC_FIRMWARE_PENDING;
 
-	switch (huc->fw.fetch_status) {
-	case INTEL_UC_FIRMWARE_FAIL:
-		/* something went wrong :( */
-		err = -EIO;
-		goto fail;
-
-	case INTEL_UC_FIRMWARE_NONE:
-	case INTEL_UC_FIRMWARE_PENDING:
-	default:
-		/* "can't happen" */
-		WARN_ONCE(1, "HuC fw %s invalid fetch_status %s [%d]\n",
-			huc->fw.path,
-			intel_uc_fw_status_repr(huc->fw.fetch_status),
-			huc->fw.fetch_status);
-		err = -ENXIO;
-		goto fail;
-
-	case INTEL_UC_FIRMWARE_SUCCESS:
-		break;
-	}
-
 	err = huc_ucode_xfer(dev_priv);
-	if (err)
-		goto fail;
 
-	huc->fw.load_status = INTEL_UC_FIRMWARE_SUCCESS;
+	huc->fw.load_status = err ?
+		INTEL_UC_FIRMWARE_FAIL : INTEL_UC_FIRMWARE_SUCCESS;
 
 	DRM_DEBUG_DRIVER("%s fw status: fetch %s, load %s\n",
 		huc->fw.path,
 		intel_uc_fw_status_repr(huc->fw.fetch_status),
 		intel_uc_fw_status_repr(huc->fw.load_status));
 
-	return 0;
-
-fail:
-	if (huc->fw.load_status == INTEL_UC_FIRMWARE_PENDING)
-		huc->fw.load_status = INTEL_UC_FIRMWARE_FAIL;
-
-	DRM_ERROR("Failed to complete HuC uCode load with ret %d\n", err);
+	if (huc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
+		DRM_ERROR("Failed to complete HuC uCode load with ret %d\n", err);
 
-	return err;
+	return;
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index b6401e8f1bd6..3c9e00d4ba5a 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -68,11 +68,20 @@ static const struct gmbus_pin gmbus_pins_bxt[] = {
 	[GMBUS_PIN_3_BXT] = { "misc", GPIOD },
 };
 
+static const struct gmbus_pin gmbus_pins_cnp[] = {
+	[GMBUS_PIN_1_BXT] = { "dpb", GPIOB },
+	[GMBUS_PIN_2_BXT] = { "dpc", GPIOC },
+	[GMBUS_PIN_3_BXT] = { "misc", GPIOD },
+	[GMBUS_PIN_4_CNP] = { "dpd", GPIOE },
+};
+
 /* pin is expected to be valid */
 static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *dev_priv,
 					     unsigned int pin)
 {
-	if (IS_GEN9_LP(dev_priv))
+	if (HAS_PCH_CNP(dev_priv))
+		return &gmbus_pins_cnp[pin];
+	else if (IS_GEN9_LP(dev_priv))
 		return &gmbus_pins_bxt[pin];
 	else if (IS_GEN9_BC(dev_priv))
 		return &gmbus_pins_skl[pin];
@@ -87,7 +96,9 @@ bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv,
 {
 	unsigned int size;
 
-	if (IS_GEN9_LP(dev_priv))
+	if (HAS_PCH_CNP(dev_priv))
+		size = ARRAY_SIZE(gmbus_pins_cnp);
+	else if (IS_GEN9_LP(dev_priv))
 		size = ARRAY_SIZE(gmbus_pins_bxt);
 	else if (IS_GEN9_BC(dev_priv))
 		size = ARRAY_SIZE(gmbus_pins_skl);
diff --git a/drivers/gpu/drm/i915/intel_lpe_audio.c b/drivers/gpu/drm/i915/intel_lpe_audio.c
index 292fedf30b00..3bf65288ffff 100644
--- a/drivers/gpu/drm/i915/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/intel_lpe_audio.c
@@ -111,6 +111,11 @@ lpe_audio_platdev_create(struct drm_i915_private *dev_priv)
 	pinfo.size_data = sizeof(*pdata);
 	pinfo.dma_mask = DMA_BIT_MASK(32);
 
+	pdata->num_pipes = INTEL_INFO(dev_priv)->num_pipes;
+	pdata->num_ports = IS_CHERRYVIEW(dev_priv) ? 3 : 2; /* B,C,D or B,C */
+	pdata->port[0].pipe = -1;
+	pdata->port[1].pipe = -1;
+	pdata->port[2].pipe = -1;
 	spin_lock_init(&pdata->lpe_audio_slock);
 
 	platdev = platform_device_register_full(&pinfo);
@@ -306,53 +311,47 @@ void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv)
  * intel_lpe_audio_notify() - notify lpe audio event
  * audio driver and i915
  * @dev_priv: the i915 drm device private data
+ * @pipe: pipe
+ * @port: port
  * @eld : ELD data
- * @pipe: pipe id
- * @port: port id
- * @tmds_clk_speed: tmds clock frequency in Hz
+ * @ls_clock: Link symbol clock in kHz
+ * @dp_output: Driving a DP output?
  *
  * Notify lpe audio driver of eld change.
  */
 void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
-			    void *eld, int port, int pipe, int tmds_clk_speed,
-			    bool dp_output, int link_rate)
+			    enum pipe pipe, enum port port,
+			    const void *eld, int ls_clock, bool dp_output)
 {
-	unsigned long irq_flags;
-	struct intel_hdmi_lpe_audio_pdata *pdata = NULL;
+	unsigned long irqflags;
+	struct intel_hdmi_lpe_audio_pdata *pdata;
+	struct intel_hdmi_lpe_audio_port_pdata *ppdata;
 	u32 audio_enable;
 
 	if (!HAS_LPE_AUDIO(dev_priv))
 		return;
 
-	pdata = dev_get_platdata(
-		&(dev_priv->lpe_audio.platdev->dev));
+	pdata = dev_get_platdata(&dev_priv->lpe_audio.platdev->dev);
+	ppdata = &pdata->port[port - PORT_B];
 
-	spin_lock_irqsave(&pdata->lpe_audio_slock, irq_flags);
+	spin_lock_irqsave(&pdata->lpe_audio_slock, irqflags);
 
 	audio_enable = I915_READ(VLV_AUD_PORT_EN_DBG(port));
 
 	if (eld != NULL) {
-		memcpy(pdata->eld.eld_data, eld,
-			HDMI_MAX_ELD_BYTES);
-		pdata->eld.port_id = port;
-		pdata->eld.pipe_id = pipe;
-		pdata->hdmi_connected = true;
-
-		pdata->dp_output = dp_output;
-		if (tmds_clk_speed)
-			pdata->tmds_clock_speed = tmds_clk_speed;
-		if (link_rate)
-			pdata->link_rate = link_rate;
+		memcpy(ppdata->eld, eld, HDMI_MAX_ELD_BYTES);
+		ppdata->pipe = pipe;
+		ppdata->ls_clock = ls_clock;
+		ppdata->dp_output = dp_output;
 
 		/* Unmute the amp for both DP and HDMI */
 		I915_WRITE(VLV_AUD_PORT_EN_DBG(port),
 			   audio_enable & ~VLV_AMP_MUTE);
-
 	} else {
-		memset(pdata->eld.eld_data, 0,
-			HDMI_MAX_ELD_BYTES);
-		pdata->hdmi_connected = false;
-		pdata->dp_output = false;
+		memset(ppdata->eld, 0, HDMI_MAX_ELD_BYTES);
+		ppdata->pipe = -1;
+		ppdata->ls_clock = 0;
+		ppdata->dp_output = false;
 
 		/* Mute the amp for both DP and HDMI */
 		I915_WRITE(VLV_AUD_PORT_EN_DBG(port),
@@ -360,10 +359,7 @@ void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
 	}
 
 	if (pdata->notify_audio_lpe)
-		pdata->notify_audio_lpe(dev_priv->lpe_audio.platdev);
-	else
-		pdata->notify_pending = true;
+		pdata->notify_audio_lpe(dev_priv->lpe_audio.platdev, port - PORT_B);
 
-	spin_unlock_irqrestore(&pdata->lpe_audio_slock,
-			irq_flags);
+	spin_unlock_irqrestore(&pdata->lpe_audio_slock, irqflags);
 }
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 62f44d3e7c43..7404cf2aac28 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -138,10 +138,6 @@
 #include "i915_drv.h"
 #include "intel_mocs.h"
 
-#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
-#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
-#define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
-
 #define RING_EXECLIST_QFULL		(1 << 0x2)
 #define RING_EXECLIST1_VALID		(1 << 0x3)
 #define RING_EXECLIST0_VALID		(1 << 0x4)
@@ -208,6 +204,7 @@
 
 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x17
 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x26
+#define GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x19
 
 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
@@ -341,39 +338,32 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq)
 
 static void execlists_submit_ports(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
 	struct execlist_port *port = engine->execlist_port;
 	u32 __iomem *elsp =
-		dev_priv->regs + i915_mmio_reg_offset(RING_ELSP(engine));
-	u64 desc[2];
-
-	GEM_BUG_ON(port[0].count > 1);
-	if (!port[0].count)
-		execlists_context_status_change(port[0].request,
-						INTEL_CONTEXT_SCHEDULE_IN);
-	desc[0] = execlists_update_context(port[0].request);
-	GEM_DEBUG_EXEC(port[0].context_id = upper_32_bits(desc[0]));
-	port[0].count++;
-
-	if (port[1].request) {
-		GEM_BUG_ON(port[1].count);
-		execlists_context_status_change(port[1].request,
-						INTEL_CONTEXT_SCHEDULE_IN);
-		desc[1] = execlists_update_context(port[1].request);
-		GEM_DEBUG_EXEC(port[1].context_id = upper_32_bits(desc[1]));
-		port[1].count = 1;
-	} else {
-		desc[1] = 0;
-	}
-	GEM_BUG_ON(desc[0] == desc[1]);
-
-	/* You must always write both descriptors in the order below. */
-	writel(upper_32_bits(desc[1]), elsp);
-	writel(lower_32_bits(desc[1]), elsp);
+		engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine));
+	unsigned int n;
+
+	for (n = ARRAY_SIZE(engine->execlist_port); n--; ) {
+		struct drm_i915_gem_request *rq;
+		unsigned int count;
+		u64 desc;
+
+		rq = port_unpack(&port[n], &count);
+		if (rq) {
+			GEM_BUG_ON(count > !n);
+			if (!count++)
+				execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
+			port_set(&port[n], port_pack(rq, count));
+			desc = execlists_update_context(rq);
+			GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
+		} else {
+			GEM_BUG_ON(!n);
+			desc = 0;
+		}
 
-	writel(upper_32_bits(desc[0]), elsp);
-	/* The context is automatically loaded after the following */
-	writel(lower_32_bits(desc[0]), elsp);
+		writel(upper_32_bits(desc), elsp);
+		writel(lower_32_bits(desc), elsp);
+	}
 }
 
 static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
@@ -394,6 +384,17 @@ static bool can_merge_ctx(const struct i915_gem_context *prev,
 	return true;
 }
 
+static void port_assign(struct execlist_port *port,
+			struct drm_i915_gem_request *rq)
+{
+	GEM_BUG_ON(rq == port_request(port));
+
+	if (port_isset(port))
+		i915_gem_request_put(port_request(port));
+
+	port_set(port, port_pack(i915_gem_request_get(rq), port_count(port)));
+}
+
 static void execlists_dequeue(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_request *last;
@@ -401,7 +402,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	struct rb_node *rb;
 	bool submit = false;
 
-	last = port->request;
+	last = port_request(port);
 	if (last)
 		/* WaIdleLiteRestore:bdw,skl
 		 * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL
@@ -411,7 +412,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		 */
 		last->tail = last->wa_tail;
 
-	GEM_BUG_ON(port[1].request);
+	GEM_BUG_ON(port_isset(&port[1]));
 
 	/* Hardware submission is through 2 ports. Conceptually each port
 	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
@@ -436,72 +437,86 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 
 	spin_lock_irq(&engine->timeline->lock);
 	rb = engine->execlist_first;
+	GEM_BUG_ON(rb_first(&engine->execlist_queue) != rb);
 	while (rb) {
-		struct drm_i915_gem_request *cursor =
-			rb_entry(rb, typeof(*cursor), priotree.node);
-
-		/* Can we combine this request with the current port? It has to
-		 * be the same context/ringbuffer and not have any exceptions
-		 * (e.g. GVT saying never to combine contexts).
-		 *
-		 * If we can combine the requests, we can execute both by
-		 * updating the RING_TAIL to point to the end of the second
-		 * request, and so we never need to tell the hardware about
-		 * the first.
-		 */
-		if (last && !can_merge_ctx(cursor->ctx, last->ctx)) {
-			/* If we are on the second port and cannot combine
-			 * this request with the last, then we are done.
-			 */
-			if (port != engine->execlist_port)
-				break;
-
-			/* If GVT overrides us we only ever submit port[0],
-			 * leaving port[1] empty. Note that we also have
-			 * to be careful that we don't queue the same
-			 * context (even though a different request) to
-			 * the second port.
+		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
+		struct drm_i915_gem_request *rq, *rn;
+
+		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
+			/*
+			 * Can we combine this request with the current port?
+			 * It has to be the same context/ringbuffer and not
+			 * have any exceptions (e.g. GVT saying never to
+			 * combine contexts).
+			 *
+			 * If we can combine the requests, we can execute both
+			 * by updating the RING_TAIL to point to the end of the
+			 * second request, and so we never need to tell the
+			 * hardware about the first.
 			 */
-			if (ctx_single_port_submission(last->ctx) ||
-			    ctx_single_port_submission(cursor->ctx))
-				break;
+			if (last && !can_merge_ctx(rq->ctx, last->ctx)) {
+				/*
+				 * If we are on the second port and cannot
+				 * combine this request with the last, then we
+				 * are done.
+				 */
+				if (port != engine->execlist_port) {
+					__list_del_many(&p->requests,
+							&rq->priotree.link);
+					goto done;
+				}
+
+				/*
+				 * If GVT overrides us we only ever submit
+				 * port[0], leaving port[1] empty. Note that we
+				 * also have to be careful that we don't queue
+				 * the same context (even though a different
+				 * request) to the second port.
+				 */
+				if (ctx_single_port_submission(last->ctx) ||
+				    ctx_single_port_submission(rq->ctx)) {
+					__list_del_many(&p->requests,
+							&rq->priotree.link);
+					goto done;
+				}
+
+				GEM_BUG_ON(last->ctx == rq->ctx);
+
+				if (submit)
+					port_assign(port, last);
+				port++;
+			}
 
-			GEM_BUG_ON(last->ctx == cursor->ctx);
+			INIT_LIST_HEAD(&rq->priotree.link);
+			rq->priotree.priority = INT_MAX;
 
-			i915_gem_request_assign(&port->request, last);
-			port++;
+			__i915_gem_request_submit(rq);
+			trace_i915_gem_request_in(rq, port_index(port, engine));
+			last = rq;
+			submit = true;
 		}
 
 		rb = rb_next(rb);
-		rb_erase(&cursor->priotree.node, &engine->execlist_queue);
-		RB_CLEAR_NODE(&cursor->priotree.node);
-		cursor->priotree.priority = INT_MAX;
-
-		__i915_gem_request_submit(cursor);
-		trace_i915_gem_request_in(cursor, port - engine->execlist_port);
-		last = cursor;
-		submit = true;
-	}
-	if (submit) {
-		i915_gem_request_assign(&port->request, last);
-		engine->execlist_first = rb;
+		rb_erase(&p->node, &engine->execlist_queue);
+		INIT_LIST_HEAD(&p->requests);
+		if (p->priority != I915_PRIORITY_NORMAL)
+			kmem_cache_free(engine->i915->priorities, p);
 	}
+done:
+	engine->execlist_first = rb;
+	if (submit)
+		port_assign(port, last);
 	spin_unlock_irq(&engine->timeline->lock);
 
 	if (submit)
 		execlists_submit_ports(engine);
 }
 
-static bool execlists_elsp_idle(struct intel_engine_cs *engine)
-{
-	return !engine->execlist_port[0].request;
-}
-
 static bool execlists_elsp_ready(const struct intel_engine_cs *engine)
 {
 	const struct execlist_port *port = engine->execlist_port;
 
-	return port[0].count + port[1].count < 2;
+	return port_count(&port[0]) + port_count(&port[1]) < 2;
 }
 
 /*
@@ -514,6 +529,15 @@ static void intel_lrc_irq_handler(unsigned long data)
 	struct execlist_port *port = engine->execlist_port;
 	struct drm_i915_private *dev_priv = engine->i915;
 
+	/* We can skip acquiring intel_runtime_pm_get() here as it was taken
+	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
+	 * not be relinquished until the device is idle (see
+	 * i915_gem_idle_work_handler()). As a precaution, we make sure
+	 * that all ELSP are drained i.e. we have processed the CSB,
+	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
+	 */
+	GEM_BUG_ON(!dev_priv->gt.awake);
+
 	intel_uncore_forcewake_get(dev_priv, engine->fw_domains);
 
 	/* Prefer doing test_and_clear_bit() as a two stage operation to avoid
@@ -542,7 +566,9 @@ static void intel_lrc_irq_handler(unsigned long data)
 		tail = GEN8_CSB_WRITE_PTR(head);
 		head = GEN8_CSB_READ_PTR(head);
 		while (head != tail) {
+			struct drm_i915_gem_request *rq;
 			unsigned int status;
+			unsigned int count;
 
 			if (++head == GEN8_CSB_ENTRIES)
 				head = 0;
@@ -570,22 +596,26 @@ static void intel_lrc_irq_handler(unsigned long data)
 
 			/* Check the context/desc id for this event matches */
 			GEM_DEBUG_BUG_ON(readl(buf + 2 * head + 1) !=
-					 port[0].context_id);
+					 port->context_id);
 
-			GEM_BUG_ON(port[0].count == 0);
-			if (--port[0].count == 0) {
+			rq = port_unpack(port, &count);
+			GEM_BUG_ON(count == 0);
+			if (--count == 0) {
 				GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
-				GEM_BUG_ON(!i915_gem_request_completed(port[0].request));
-				execlists_context_status_change(port[0].request,
-								INTEL_CONTEXT_SCHEDULE_OUT);
+				GEM_BUG_ON(!i915_gem_request_completed(rq));
+				execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
+
+				trace_i915_gem_request_out(rq);
+				i915_gem_request_put(rq);
 
-				trace_i915_gem_request_out(port[0].request);
-				i915_gem_request_put(port[0].request);
 				port[0] = port[1];
 				memset(&port[1], 0, sizeof(port[1]));
+			} else {
+				port_set(port, port_pack(rq, count));
 			}
 
-			GEM_BUG_ON(port[0].count == 0 &&
+			/* After the final element, the hw should be idle */
+			GEM_BUG_ON(port_count(port) == 0 &&
 				   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
 		}
 
@@ -599,28 +629,66 @@ static void intel_lrc_irq_handler(unsigned long data)
 	intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
 }
 
-static bool insert_request(struct i915_priotree *pt, struct rb_root *root)
+static bool
+insert_request(struct intel_engine_cs *engine,
+	       struct i915_priotree *pt,
+	       int prio)
 {
-	struct rb_node **p, *rb;
+	struct i915_priolist *p;
+	struct rb_node **parent, *rb;
 	bool first = true;
 
+	if (unlikely(engine->no_priolist))
+		prio = I915_PRIORITY_NORMAL;
+
+find_priolist:
 	/* most positive priority is scheduled first, equal priorities fifo */
 	rb = NULL;
-	p = &root->rb_node;
-	while (*p) {
-		struct i915_priotree *pos;
-
-		rb = *p;
-		pos = rb_entry(rb, typeof(*pos), node);
-		if (pt->priority > pos->priority) {
-			p = &rb->rb_left;
-		} else {
-			p = &rb->rb_right;
+	parent = &engine->execlist_queue.rb_node;
+	while (*parent) {
+		rb = *parent;
+		p = rb_entry(rb, typeof(*p), node);
+		if (prio > p->priority) {
+			parent = &rb->rb_left;
+		} else if (prio < p->priority) {
+			parent = &rb->rb_right;
 			first = false;
+		} else {
+			list_add_tail(&pt->link, &p->requests);
+			return false;
 		}
 	}
-	rb_link_node(&pt->node, rb, p);
-	rb_insert_color(&pt->node, root);
+
+	if (prio == I915_PRIORITY_NORMAL) {
+		p = &engine->default_priolist;
+	} else {
+		p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
+		/* Convert an allocation failure to a priority bump */
+		if (unlikely(!p)) {
+			prio = I915_PRIORITY_NORMAL; /* recurses just once */
+
+			/* To maintain ordering with all rendering, after an
+			 * allocation failure we have to disable all scheduling.
+			 * Requests will then be executed in fifo, and schedule
+			 * will ensure that dependencies are emitted in fifo.
+			 * There will be still some reordering with existing
+			 * requests, so if userspace lied about their
+			 * dependencies that reordering may be visible.
+			 */
+			engine->no_priolist = true;
+			goto find_priolist;
+		}
+	}
+
+	p->priority = prio;
+	rb_link_node(&p->node, rb, parent);
+	rb_insert_color(&p->node, &engine->execlist_queue);
+
+	INIT_LIST_HEAD(&p->requests);
+	list_add_tail(&pt->link, &p->requests);
+
+	if (first)
+		engine->execlist_first = &p->node;
 
 	return first;
 }
@@ -633,12 +701,16 @@ static void execlists_submit_request(struct drm_i915_gem_request *request)
 	/* Will be called from irq-context when using foreign fences. */
 	spin_lock_irqsave(&engine->timeline->lock, flags);
 
-	if (insert_request(&request->priotree, &engine->execlist_queue)) {
-		engine->execlist_first = &request->priotree.node;
+	if (insert_request(engine,
+			   &request->priotree,
+			   request->priotree.priority)) {
 		if (execlists_elsp_ready(engine))
 			tasklet_hi_schedule(&engine->irq_tasklet);
 	}
 
+	GEM_BUG_ON(!engine->execlist_first);
+	GEM_BUG_ON(list_empty(&request->priotree.link));
+
 	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 }
 
@@ -708,6 +780,19 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
 		list_safe_reset_next(dep, p, dfs_link);
 	}
 
+	/* If we didn't need to bump any existing priorities, and we haven't
+	 * yet submitted this request (i.e. there is no potential race with
+	 * execlists_submit_request()), we can set our own priority and skip
+	 * acquiring the engine locks.
+	 */
+	if (request->priotree.priority == INT_MIN) {
+		GEM_BUG_ON(!list_empty(&request->priotree.link));
+		request->priotree.priority = prio;
+		if (stack.dfs_link.next == stack.dfs_link.prev)
+			return;
+		__list_del_entry(&stack.dfs_link);
+	}
+
 	engine = request->engine;
 	spin_lock_irq(&engine->timeline->lock);
 
@@ -723,10 +808,9 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
 			continue;
 
 		pt->priority = prio;
-		if (!RB_EMPTY_NODE(&pt->node)) {
-			rb_erase(&pt->node, &engine->execlist_queue);
-			if (insert_request(pt, &engine->execlist_queue))
-				engine->execlist_first = &pt->node;
+		if (!list_empty(&pt->link)) {
+			__list_del_entry(&pt->link);
+			insert_request(engine, pt, prio);
 		}
 	}
 
@@ -735,8 +819,9 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
 	/* XXX Do we need to preempt to make room for us and our deps? */
 }
 
-static int execlists_context_pin(struct intel_engine_cs *engine,
-				 struct i915_gem_context *ctx)
+static struct intel_ring *
+execlists_context_pin(struct intel_engine_cs *engine,
+		      struct i915_gem_context *ctx)
 {
 	struct intel_context *ce = &ctx->engine[engine->id];
 	unsigned int flags;
@@ -745,8 +830,8 @@ static int execlists_context_pin(struct intel_engine_cs *engine,
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 
-	if (ce->pin_count++)
-		return 0;
+	if (likely(ce->pin_count++))
+		goto out;
 	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
 
 	if (!ce->state) {
@@ -770,7 +855,7 @@ static int execlists_context_pin(struct intel_engine_cs *engine,
 		goto unpin_vma;
 	}
 
-	ret = intel_ring_pin(ce->ring, ctx->ggtt_offset_bias);
+	ret = intel_ring_pin(ce->ring, ctx->i915, ctx->ggtt_offset_bias);
 	if (ret)
 		goto unpin_map;
 
@@ -783,7 +868,8 @@ static int execlists_context_pin(struct intel_engine_cs *engine,
 	ce->state->obj->mm.dirty = true;
 
 	i915_gem_context_get(ctx);
-	return 0;
+out:
+	return ce->ring;
 
 unpin_map:
 	i915_gem_object_unpin_map(ce->state->obj);
@@ -791,7 +877,7 @@ unpin_vma:
 	__i915_vma_unpin(ce->state);
 err:
 	ce->pin_count = 0;
-	return ret;
+	return ERR_PTR(ret);
 }
 
 static void execlists_context_unpin(struct intel_engine_cs *engine,
@@ -828,9 +914,6 @@ static int execlists_request_alloc(struct drm_i915_gem_request *request)
 	 */
 	request->reserved_space += EXECLISTS_REQUEST_SIZE;
 
-	GEM_BUG_ON(!ce->ring);
-	request->ring = ce->ring;
-
 	if (i915.enable_guc_submission) {
 		/*
 		 * Check that the GuC has space for the request before
@@ -1138,14 +1221,12 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 	return ret;
 }
 
-static u32 port_seqno(struct execlist_port *port)
-{
-	return port->request ? port->request->global_seqno : 0;
-}
-
 static int gen8_init_common_ring(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
+	struct execlist_port *port = engine->execlist_port;
+	unsigned int n;
+	bool submit;
 	int ret;
 
 	ret = intel_mocs_init_engine(engine);
@@ -1166,16 +1247,24 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 
 	/* After a GPU reset, we may have requests to replay */
 	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-	if (!i915.enable_guc_submission && !execlists_elsp_idle(engine)) {
-		DRM_DEBUG_DRIVER("Restarting %s from requests [0x%x, 0x%x]\n",
-				 engine->name,
-				 port_seqno(&engine->execlist_port[0]),
-				 port_seqno(&engine->execlist_port[1]));
-		engine->execlist_port[0].count = 0;
-		engine->execlist_port[1].count = 0;
-		execlists_submit_ports(engine);
+
+	submit = false;
+	for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++) {
+		if (!port_isset(&port[n]))
+			break;
+
+		DRM_DEBUG_DRIVER("Restarting %s:%d from 0x%x\n",
+				 engine->name, n,
+				 port_request(&port[n])->global_seqno);
+
+		/* Discard the current inflight count */
+		port_set(&port[n], port_request(&port[n]));
+		submit = true;
 	}
 
+	if (submit && !i915.enable_guc_submission)
+		execlists_submit_ports(engine);
+
 	return 0;
 }
 
@@ -1251,13 +1340,13 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	intel_ring_update_space(request->ring);
 
 	/* Catch up with any missed context-switch interrupts */
-	if (request->ctx != port[0].request->ctx) {
-		i915_gem_request_put(port[0].request);
+	if (request->ctx != port_request(port)->ctx) {
+		i915_gem_request_put(port_request(port));
 		port[0] = port[1];
 		memset(&port[1], 0, sizeof(port[1]));
 	}
 
-	GEM_BUG_ON(request->ctx != port[0].request->ctx);
+	GEM_BUG_ON(request->ctx != port_request(port)->ctx);
 
 	/* Reset WaIdleLiteRestore:bdw,skl as well */
 	request->tail =
@@ -1773,6 +1862,10 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
 	default:
 		MISSING_CASE(INTEL_GEN(engine->i915));
 		/* fall through */
+	case 10:
+		indirect_ctx_offset =
+			GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
+		break;
 	case 9:
 		indirect_ctx_offset =
 			GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
@@ -1869,6 +1962,8 @@ static void execlists_init_reg_state(u32 *regs,
 		regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
 		CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
 			make_rpcs(dev_priv));
+
+		i915_oa_init_reg_state(engine, ctx, regs);
 	}
 }
 
@@ -1906,44 +2001,6 @@ populate_lr_context(struct i915_gem_context *ctx,
 	return 0;
 }
 
-/**
- * intel_lr_context_size() - return the size of the context for an engine
- * @engine: which engine to find the context size for
- *
- * Each engine may require a different amount of space for a context image,
- * so when allocating (or copying) an image, this function can be used to
- * find the right size for the specific engine.
- *
- * Return: size (in bytes) of an engine-specific context image
- *
- * Note: this size includes the HWSP, which is part of the context image
- * in LRC mode, but does not include the "shared data page" used with
- * GuC submission. The caller should account for this if using the GuC.
- */
-uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
-{
-	int ret = 0;
-
-	WARN_ON(INTEL_GEN(engine->i915) < 8);
-
-	switch (engine->id) {
-	case RCS:
-		if (INTEL_GEN(engine->i915) >= 9)
-			ret = GEN9_LR_CONTEXT_RENDER_SIZE;
-		else
-			ret = GEN8_LR_CONTEXT_RENDER_SIZE;
-		break;
-	case VCS:
-	case BCS:
-	case VECS:
-	case VCS2:
-		ret = GEN8_LR_CONTEXT_OTHER_SIZE;
-		break;
-	}
-
-	return ret;
-}
-
 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 					    struct intel_engine_cs *engine)
 {
@@ -1956,8 +2013,7 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 
 	WARN_ON(ce->state);
 
-	context_size = round_up(intel_lr_context_size(engine),
-				I915_GTT_PAGE_SIZE);
+	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);
 
 	/* One extra page as the sharing data between driver and GuC */
 	context_size += PAGE_SIZE * LRC_PPHWSP_PN;
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index e8015e7bf4e9..52b3a1fd4059 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -78,8 +78,6 @@ int logical_xcs_ring_init(struct intel_engine_cs *engine);
 struct drm_i915_private;
 struct i915_gem_context;
 
-uint32_t intel_lr_context_size(struct intel_engine_cs *engine);
-
 void intel_lr_context_resume(struct drm_i915_private *dev_priv);
 uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
 				     struct intel_engine_cs *engine);
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 8b942ef2b3ec..6fe5d7c3bc23 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -311,8 +311,6 @@ static void intel_enable_lvds(struct intel_encoder *encoder,
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-	struct intel_connector *intel_connector =
-		&lvds_encoder->attached_connector->base;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
 	I915_WRITE(lvds_encoder->reg, I915_READ(lvds_encoder->reg) | LVDS_PORT_EN);
@@ -322,7 +320,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder,
 	if (intel_wait_for_register(dev_priv, PP_STATUS(0), PP_ON, PP_ON, 1000))
 		DRM_ERROR("timed out waiting for panel to power on\n");
 
-	intel_panel_enable_backlight(intel_connector);
+	intel_panel_enable_backlight(pipe_config, conn_state);
 }
 
 static void intel_disable_lvds(struct intel_encoder *encoder,
@@ -345,11 +343,7 @@ static void gmch_disable_lvds(struct intel_encoder *encoder,
 			      struct drm_connector_state *old_conn_state)
 
 {
-	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-	struct intel_connector *intel_connector =
-		&lvds_encoder->attached_connector->base;
-
-	intel_panel_disable_backlight(intel_connector);
+	intel_panel_disable_backlight(old_conn_state);
 
 	intel_disable_lvds(encoder, old_crtc_state, old_conn_state);
 }
@@ -358,11 +352,7 @@ static void pch_disable_lvds(struct intel_encoder *encoder,
 			     struct intel_crtc_state *old_crtc_state,
 			     struct drm_connector_state *old_conn_state)
 {
-	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-	struct intel_connector *intel_connector =
-		&lvds_encoder->attached_connector->base;
-
-	intel_panel_disable_backlight(intel_connector);
+	intel_panel_disable_backlight(old_conn_state);
 }
 
 static void pch_post_disable_lvds(struct intel_encoder *encoder,
@@ -433,10 +423,10 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 		pipe_config->has_pch_encoder = true;
 
 		intel_pch_panel_fitting(intel_crtc, pipe_config,
-					intel_connector->panel.fitting_mode);
+					conn_state->scaling_mode);
 	} else {
 		intel_gmch_panel_fitting(intel_crtc, pipe_config,
-					 intel_connector->panel.fitting_mode);
+					 conn_state->scaling_mode);
 
 	}
 
@@ -598,56 +588,24 @@ static void intel_lvds_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-static int intel_lvds_set_property(struct drm_connector *connector,
-				   struct drm_property *property,
-				   uint64_t value)
-{
-	struct intel_connector *intel_connector = to_intel_connector(connector);
-	struct drm_device *dev = connector->dev;
-
-	if (property == dev->mode_config.scaling_mode_property) {
-		struct drm_crtc *crtc;
-
-		if (value == DRM_MODE_SCALE_NONE) {
-			DRM_DEBUG_KMS("no scaling not supported\n");
-			return -EINVAL;
-		}
-
-		if (intel_connector->panel.fitting_mode == value) {
-			/* the LVDS scaling property is not changed */
-			return 0;
-		}
-		intel_connector->panel.fitting_mode = value;
-
-		crtc = intel_attached_encoder(connector)->base.crtc;
-		if (crtc && crtc->state->enable) {
-			/*
-			 * If the CRTC is enabled, the display will be changed
-			 * according to the new panel fitting mode.
-			 */
-			intel_crtc_restore_mode(crtc);
-		}
-	}
-
-	return 0;
-}
-
 static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = {
 	.get_modes = intel_lvds_get_modes,
 	.mode_valid = intel_lvds_mode_valid,
+	.atomic_check = intel_digital_connector_atomic_check,
 };
 
 static const struct drm_connector_funcs intel_lvds_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = intel_lvds_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.atomic_get_property = intel_digital_connector_atomic_get_property,
+	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_lvds_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
 };
 
 static const struct drm_encoder_funcs intel_lvds_enc_funcs = {
@@ -988,6 +946,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	u32 lvds;
 	int pipe;
 	u8 pin;
+	u32 allowed_scalers;
 
 	if (!intel_lvds_supported(dev_priv))
 		return;
@@ -1083,11 +1042,11 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	lvds_encoder->reg = lvds_reg;
 
 	/* create the scaling mode property */
-	drm_mode_create_scaling_mode_property(dev);
-	drm_object_attach_property(&connector->base,
-				      dev->mode_config.scaling_mode_property,
-				      DRM_MODE_SCALE_ASPECT);
-	intel_connector->panel.fitting_mode = DRM_MODE_SCALE_ASPECT;
+	allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT);
+	allowed_scalers |= BIT(DRM_MODE_SCALE_FULLSCREEN);
+	allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER);
+	drm_connector_attach_scaling_mode_property(connector, allowed_scalers);
+	connector->state->scaling_mode = DRM_MODE_SCALE_ASPECT;
 
 	intel_lvds_pps_get_hw_state(dev_priv, &lvds_encoder->init_pps);
 	lvds_encoder->init_lvds_val = lvds;
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index 92e461c68385..f4c46b0b8f0a 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -178,7 +178,7 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv,
 {
 	bool result = false;
 
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		table->size  = ARRAY_SIZE(skylake_mocs_table);
 		table->table = skylake_mocs_table;
 		result = true;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index d44465190dc1..2bd03001cc70 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -461,7 +461,7 @@ static u32 asle_set_backlight(struct drm_i915_private *dev_priv, u32 bclp)
 	DRM_DEBUG_KMS("updating opregion backlight %d/255\n", bclp);
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	for_each_intel_connector_iter(connector, &conn_iter)
-		intel_panel_set_backlight_acpi(connector, bclp, 255);
+		intel_panel_set_backlight_acpi(connector->base.state, bclp, 255);
 	drm_connector_list_iter_end(&conn_iter);
 	asle->cblv = DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID;
 
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 2e0c56ed22bb..b96aed941b97 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -270,7 +270,6 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 	u32 *cs;
 
 	WARN_ON(overlay->active);
-	WARN_ON(IS_I830(dev_priv) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE));
 
 	req = alloc_request(overlay);
 	if (IS_ERR(req))
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index cb50c527401f..96c2cbd81869 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -561,15 +561,18 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector)
 	return val;
 }
 
-static void lpt_set_backlight(struct intel_connector *connector, u32 level)
+static void lpt_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+
 	u32 val = I915_READ(BLC_PWM_PCH_CTL2) & ~BACKLIGHT_DUTY_CYCLE_MASK;
 	I915_WRITE(BLC_PWM_PCH_CTL2, val | level);
 }
 
-static void pch_set_backlight(struct intel_connector *connector, u32 level)
+static void pch_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
@@ -577,8 +580,9 @@ static void pch_set_backlight(struct intel_connector *connector, u32 level)
 	I915_WRITE(BLC_PWM_CPU_CTL, tmp | level);
 }
 
-static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
+static void i9xx_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 tmp, mask;
@@ -604,50 +608,51 @@ static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
 	I915_WRITE(BLC_PWM_CTL, tmp | level);
 }
 
-static void vlv_set_backlight(struct intel_connector *connector, u32 level)
+static void vlv_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
+	enum pipe pipe = to_intel_crtc(conn_state->crtc)->pipe;
 	u32 tmp;
 
-	if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B))
-		return;
-
 	tmp = I915_READ(VLV_BLC_PWM_CTL(pipe)) & ~BACKLIGHT_DUTY_CYCLE_MASK;
 	I915_WRITE(VLV_BLC_PWM_CTL(pipe), tmp | level);
 }
 
-static void bxt_set_backlight(struct intel_connector *connector, u32 level)
+static void bxt_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
 	I915_WRITE(BXT_BLC_PWM_DUTY(panel->backlight.controller), level);
 }
 
-static void pwm_set_backlight(struct intel_connector *connector, u32 level)
+static void pwm_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
-	struct intel_panel *panel = &connector->panel;
+	struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel;
 	int duty_ns = DIV_ROUND_UP(level * CRC_PMIC_PWM_PERIOD_NS, 100);
 
 	pwm_config(panel->backlight.pwm, duty_ns, CRC_PMIC_PWM_PERIOD_NS);
 }
 
 static void
-intel_panel_actually_set_backlight(struct intel_connector *connector, u32 level)
+intel_panel_actually_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct intel_panel *panel = &connector->panel;
 
 	DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level);
 
 	level = intel_panel_compute_brightness(connector, level);
-	panel->backlight.set(connector, level);
+	panel->backlight.set(conn_state, level);
 }
 
 /* set backlight brightness to level in range [0..max], scaling wrt hw min */
-static void intel_panel_set_backlight(struct intel_connector *connector,
+static void intel_panel_set_backlight(const struct drm_connector_state *conn_state,
 				      u32 user_level, u32 user_max)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 hw_level;
@@ -663,7 +668,7 @@ static void intel_panel_set_backlight(struct intel_connector *connector,
 	panel->backlight.level = hw_level;
 
 	if (panel->backlight.enabled)
-		intel_panel_actually_set_backlight(connector, hw_level);
+		intel_panel_actually_set_backlight(conn_state, hw_level);
 
 	mutex_unlock(&dev_priv->backlight_lock);
 }
@@ -671,21 +676,21 @@ static void intel_panel_set_backlight(struct intel_connector *connector,
 /* set backlight brightness to level in range [0..max], assuming hw min is
  * respected.
  */
-void intel_panel_set_backlight_acpi(struct intel_connector *connector,
+void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state,
 				    u32 user_level, u32 user_max)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 hw_level;
 
 	/*
-	 * INVALID_PIPE may occur during driver init because
+	 * Lack of crtc may occur during driver init because
 	 * connection_mutex isn't held across the entire backlight
 	 * setup + modeset readout, and the BIOS can issue the
 	 * requests at any time.
 	 */
-	if (!panel->backlight.present || pipe == INVALID_PIPE)
+	if (!panel->backlight.present || !conn_state->crtc)
 		return;
 
 	mutex_lock(&dev_priv->backlight_lock);
@@ -702,17 +707,18 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector,
 					 panel->backlight.device->props.max_brightness);
 
 	if (panel->backlight.enabled)
-		intel_panel_actually_set_backlight(connector, hw_level);
+		intel_panel_actually_set_backlight(conn_state, hw_level);
 
 	mutex_unlock(&dev_priv->backlight_lock);
 }
 
-static void lpt_disable_backlight(struct intel_connector *connector)
+static void lpt_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
-	intel_panel_actually_set_backlight(connector, 0);
+	intel_panel_actually_set_backlight(old_conn_state, 0);
 
 	/*
 	 * Although we don't support or enable CPU PWM with LPT/SPT based
@@ -732,12 +738,13 @@ static void lpt_disable_backlight(struct intel_connector *connector)
 	I915_WRITE(BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE);
 }
 
-static void pch_disable_backlight(struct intel_connector *connector)
+static void pch_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
-	intel_panel_actually_set_backlight(connector, 0);
+	intel_panel_actually_set_backlight(old_conn_state, 0);
 
 	tmp = I915_READ(BLC_PWM_CPU_CTL2);
 	I915_WRITE(BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE);
@@ -746,44 +753,43 @@ static void pch_disable_backlight(struct intel_connector *connector)
 	I915_WRITE(BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE);
 }
 
-static void i9xx_disable_backlight(struct intel_connector *connector)
+static void i9xx_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
-	intel_panel_actually_set_backlight(connector, 0);
+	intel_panel_actually_set_backlight(old_conn_state, 0);
 }
 
-static void i965_disable_backlight(struct intel_connector *connector)
+static void i965_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(old_conn_state->connector->dev);
 	u32 tmp;
 
-	intel_panel_actually_set_backlight(connector, 0);
+	intel_panel_actually_set_backlight(old_conn_state, 0);
 
 	tmp = I915_READ(BLC_PWM_CTL2);
 	I915_WRITE(BLC_PWM_CTL2, tmp & ~BLM_PWM_ENABLE);
 }
 
-static void vlv_disable_backlight(struct intel_connector *connector)
+static void vlv_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
+	enum pipe pipe = to_intel_crtc(old_conn_state->crtc)->pipe;
 	u32 tmp;
 
-	if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B))
-		return;
-
-	intel_panel_actually_set_backlight(connector, 0);
+	intel_panel_actually_set_backlight(old_conn_state, 0);
 
 	tmp = I915_READ(VLV_BLC_PWM_CTL2(pipe));
 	I915_WRITE(VLV_BLC_PWM_CTL2(pipe), tmp & ~BLM_PWM_ENABLE);
 }
 
-static void bxt_disable_backlight(struct intel_connector *connector)
+static void bxt_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 tmp, val;
 
-	intel_panel_actually_set_backlight(connector, 0);
+	intel_panel_actually_set_backlight(old_conn_state, 0);
 
 	tmp = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
 	I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
@@ -796,8 +802,23 @@ static void bxt_disable_backlight(struct intel_connector *connector)
 	}
 }
 
-static void pwm_disable_backlight(struct intel_connector *connector)
+static void cnp_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
+	u32 tmp;
+
+	intel_panel_actually_set_backlight(old_conn_state, 0);
+
+	tmp = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+	I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
+		   tmp & ~BXT_BLC_PWM_ENABLE);
+}
+
+static void pwm_disable_backlight(const struct drm_connector_state *old_conn_state)
+{
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct intel_panel *panel = &connector->panel;
 
 	/* Disable the backlight */
@@ -806,8 +827,9 @@ static void pwm_disable_backlight(struct intel_connector *connector)
 	pwm_disable(panel->backlight.pwm);
 }
 
-void intel_panel_disable_backlight(struct intel_connector *connector)
+void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
@@ -830,13 +852,15 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
 	if (panel->backlight.device)
 		panel->backlight.device->props.power = FB_BLANK_POWERDOWN;
 	panel->backlight.enabled = false;
-	panel->backlight.disable(connector);
+	panel->backlight.disable(old_conn_state);
 
 	mutex_unlock(&dev_priv->backlight_lock);
 }
 
-static void lpt_enable_backlight(struct intel_connector *connector)
+static void lpt_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 pch_ctl1, pch_ctl2, schicken;
@@ -880,16 +904,16 @@ static void lpt_enable_backlight(struct intel_connector *connector)
 	I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1 | BLM_PCH_PWM_ENABLE);
 
 	/* This won't stick until the above enable. */
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 }
 
-static void pch_enable_backlight(struct intel_connector *connector)
+static void pch_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
-	enum transcoder cpu_transcoder =
-		intel_pipe_to_cpu_transcoder(dev_priv, pipe);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	u32 cpu_ctl2, pch_ctl1, pch_ctl2;
 
 	cpu_ctl2 = I915_READ(BLC_PWM_CPU_CTL2);
@@ -915,7 +939,7 @@ static void pch_enable_backlight(struct intel_connector *connector)
 	I915_WRITE(BLC_PWM_CPU_CTL2, cpu_ctl2 | BLM_PWM_ENABLE);
 
 	/* This won't stick until the above enable. */
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 
 	pch_ctl2 = panel->backlight.max << 16;
 	I915_WRITE(BLC_PWM_PCH_CTL2, pch_ctl2);
@@ -929,8 +953,10 @@ static void pch_enable_backlight(struct intel_connector *connector)
 	I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1 | BLM_PCH_PWM_ENABLE);
 }
 
-static void i9xx_enable_backlight(struct intel_connector *connector)
+static void i9xx_enable_backlight(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 ctl, freq;
@@ -955,7 +981,7 @@ static void i9xx_enable_backlight(struct intel_connector *connector)
 	POSTING_READ(BLC_PWM_CTL);
 
 	/* XXX: combine this into above write? */
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 
 	/*
 	 * Needed to enable backlight on some 855gm models. BLC_HIST_CTL is
@@ -966,11 +992,13 @@ static void i9xx_enable_backlight(struct intel_connector *connector)
 		I915_WRITE(BLC_HIST_CTL, BLM_HISTOGRAM_ENABLE);
 }
 
-static void i965_enable_backlight(struct intel_connector *connector)
+static void i965_enable_backlight(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
+	enum pipe pipe = to_intel_crtc(conn_state->crtc)->pipe;
 	u32 ctl, ctl2, freq;
 
 	ctl2 = I915_READ(BLC_PWM_CTL2);
@@ -996,19 +1024,18 @@ static void i965_enable_backlight(struct intel_connector *connector)
 	POSTING_READ(BLC_PWM_CTL2);
 	I915_WRITE(BLC_PWM_CTL2, ctl2 | BLM_PWM_ENABLE);
 
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 }
 
-static void vlv_enable_backlight(struct intel_connector *connector)
+static void vlv_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
+	enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
 	u32 ctl, ctl2;
 
-	if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B))
-		return;
-
 	ctl2 = I915_READ(VLV_BLC_PWM_CTL2(pipe));
 	if (ctl2 & BLM_PWM_ENABLE) {
 		DRM_DEBUG_KMS("backlight already enabled\n");
@@ -1020,7 +1047,7 @@ static void vlv_enable_backlight(struct intel_connector *connector)
 	I915_WRITE(VLV_BLC_PWM_CTL(pipe), ctl);
 
 	/* XXX: combine this into above write? */
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 
 	ctl2 = 0;
 	if (panel->backlight.active_low_pwm)
@@ -1030,11 +1057,13 @@ static void vlv_enable_backlight(struct intel_connector *connector)
 	I915_WRITE(VLV_BLC_PWM_CTL2(pipe), ctl2 | BLM_PWM_ENABLE);
 }
 
-static void bxt_enable_backlight(struct intel_connector *connector)
+static void bxt_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
+	enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
 	u32 pwm_ctl, val;
 
 	/* Controller 1 uses the utility pin. */
@@ -1064,7 +1093,7 @@ static void bxt_enable_backlight(struct intel_connector *connector)
 	I915_WRITE(BXT_BLC_PWM_FREQ(panel->backlight.controller),
 			panel->backlight.max);
 
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 
 	pwm_ctl = 0;
 	if (panel->backlight.active_low_pwm)
@@ -1076,19 +1105,54 @@ static void bxt_enable_backlight(struct intel_connector *connector)
 			pwm_ctl | BXT_BLC_PWM_ENABLE);
 }
 
-static void pwm_enable_backlight(struct intel_connector *connector)
+static void cnp_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
+	u32 pwm_ctl;
+
+	pwm_ctl = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+	if (pwm_ctl & BXT_BLC_PWM_ENABLE) {
+		DRM_DEBUG_KMS("backlight already enabled\n");
+		pwm_ctl &= ~BXT_BLC_PWM_ENABLE;
+		I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
+			   pwm_ctl);
+	}
+
+	I915_WRITE(BXT_BLC_PWM_FREQ(panel->backlight.controller),
+		   panel->backlight.max);
+
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
+
+	pwm_ctl = 0;
+	if (panel->backlight.active_low_pwm)
+		pwm_ctl |= BXT_BLC_PWM_POLARITY;
+
+	I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller), pwm_ctl);
+	POSTING_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+	I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
+		   pwm_ctl | BXT_BLC_PWM_ENABLE);
+}
+
+static void pwm_enable_backlight(const struct intel_crtc_state *crtc_state,
+				 const struct drm_connector_state *conn_state)
+{
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct intel_panel *panel = &connector->panel;
 
 	pwm_enable(panel->backlight.pwm);
-	intel_panel_actually_set_backlight(connector, panel->backlight.level);
+	intel_panel_actually_set_backlight(conn_state, panel->backlight.level);
 }
 
-void intel_panel_enable_backlight(struct intel_connector *connector)
+void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	enum pipe pipe = intel_get_pipe_from_connector(connector);
+	enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
 
 	if (!panel->backlight.present)
 		return;
@@ -1108,7 +1172,7 @@ void intel_panel_enable_backlight(struct intel_connector *connector)
 						 panel->backlight.device->props.max_brightness);
 	}
 
-	panel->backlight.enable(connector);
+	panel->backlight.enable(crtc_state, conn_state);
 	panel->backlight.enabled = true;
 	if (panel->backlight.device)
 		panel->backlight.device->props.power = FB_BLANK_UNBLANK;
@@ -1126,7 +1190,7 @@ static int intel_backlight_device_update_status(struct backlight_device *bd)
 	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
 	DRM_DEBUG_KMS("updating intel_backlight, brightness=%d/%d\n",
 		      bd->props.brightness, bd->props.max_brightness);
-	intel_panel_set_backlight(connector, bd->props.brightness,
+	intel_panel_set_backlight(connector->base.state, bd->props.brightness,
 				  bd->props.max_brightness);
 
 	/*
@@ -1239,6 +1303,17 @@ void intel_backlight_device_unregister(struct intel_connector *connector)
 #endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
 
 /*
+ * CNP: PWM clock frequency is 19.2 MHz or 24 MHz.
+ *      PWM increment = 1
+ */
+static u32 cnp_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+
+	return DIV_ROUND_CLOSEST(KHz(dev_priv->rawclk_freq), pwm_freq_hz);
+}
+
+/*
  * BXT: PWM clock frequency = 19.2 MHz.
  */
 static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
@@ -1633,6 +1708,42 @@ bxt_setup_backlight(struct intel_connector *connector, enum pipe unused)
 	return 0;
 }
 
+static int
+cnp_setup_backlight(struct intel_connector *connector, enum pipe unused)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
+	u32 pwm_ctl, val;
+
+	/*
+	 * CNP has the BXT implementation of backlight, but with only
+	 * one controller. Future platforms could have multiple controllers
+	 * so let's make this extensible and prepared for the future.
+	 */
+	panel->backlight.controller = 0;
+
+	pwm_ctl = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+
+	panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY;
+	panel->backlight.max =
+		I915_READ(BXT_BLC_PWM_FREQ(panel->backlight.controller));
+
+	if (!panel->backlight.max)
+		panel->backlight.max = get_backlight_max_vbt(connector);
+
+	if (!panel->backlight.max)
+		return -ENODEV;
+
+	val = bxt_get_backlight(connector);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
+
+	panel->backlight.enabled = pwm_ctl & BXT_BLC_PWM_ENABLE;
+
+	return 0;
+}
+
 static int pwm_setup_backlight(struct intel_connector *connector,
 			       enum pipe pipe)
 {
@@ -1749,6 +1860,13 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel)
 		panel->backlight.set = bxt_set_backlight;
 		panel->backlight.get = bxt_get_backlight;
 		panel->backlight.hz_to_pwm = bxt_hz_to_pwm;
+	} else if (HAS_PCH_CNP(dev_priv)) {
+		panel->backlight.setup = cnp_setup_backlight;
+		panel->backlight.enable = cnp_enable_backlight;
+		panel->backlight.disable = cnp_disable_backlight;
+		panel->backlight.set = bxt_set_backlight;
+		panel->backlight.get = bxt_get_backlight;
+		panel->backlight.hz_to_pwm = cnp_hz_to_pwm;
 	} else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_SPT(dev_priv) ||
 		   HAS_PCH_KBP(dev_priv)) {
 		panel->backlight.setup = lpt_setup_backlight;
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index 206ee4f0150e..8fbd2bd0877f 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -513,16 +513,20 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
 	struct intel_crtc_state *pipe_config;
 	struct drm_atomic_state *state;
+	struct drm_modeset_acquire_ctx ctx;
 	int ret = 0;
 
-	drm_modeset_lock_all(dev);
+	drm_modeset_acquire_init(&ctx, 0);
+
 	state = drm_atomic_state_alloc(dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto unlock;
 	}
 
-	state->acquire_ctx = crtc->base.dev->mode_config.acquire_ctx;
+	state->acquire_ctx = &ctx;
+
+retry:
 	pipe_config = intel_atomic_get_crtc_state(state, crtc);
 	if (IS_ERR(pipe_config)) {
 		ret = PTR_ERR(pipe_config);
@@ -537,10 +541,17 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
 	ret = drm_atomic_commit(state);
 
 put_state:
+	if (ret == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
 	drm_atomic_state_put(state);
 unlock:
 	WARN(ret, "Toggling workaround to %i returns %i\n", enable, ret);
-	drm_modeset_unlock_all(dev);
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
 }
 
 static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
@@ -842,19 +853,12 @@ static ssize_t display_crc_ctl_write(struct file *file, const char __user *ubuf,
 		return -E2BIG;
 	}
 
-	tmpbuf = kmalloc(len + 1, GFP_KERNEL);
-	if (!tmpbuf)
-		return -ENOMEM;
-
-	if (copy_from_user(tmpbuf, ubuf, len)) {
-		ret = -EFAULT;
-		goto out;
-	}
-	tmpbuf[len] = '\0';
+	tmpbuf = memdup_user_nul(ubuf, len);
+	if (IS_ERR(tmpbuf))
+		return PTR_ERR(tmpbuf);
 
 	ret = display_crc_ctl_parse(dev_priv, tmpbuf, len);
 
-out:
 	kfree(tmpbuf);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 078fd1bfa5ea..48ea0fca1f72 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -58,24 +58,24 @@
 
 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl */
+	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
 	I915_WRITE(CHICKEN_PAR1_1,
 		   I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
 
 	I915_WRITE(GEN8_CONFIG0,
 		   I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES);
 
-	/* WaEnableChickenDCPR:skl,bxt,kbl,glk */
+	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
 	I915_WRITE(GEN8_CHICKEN_DCPR_1,
 		   I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
 
-	/* WaFbcTurnOffFbcWatermark:skl,bxt,kbl */
-	/* WaFbcWakeMemOn:skl,bxt,kbl,glk */
+	/* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
+	/* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
 	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 		   DISP_FBC_WM_DIS |
 		   DISP_FBC_MEMORY_WAKE);
 
-	/* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl */
+	/* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
 	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
 		   ILK_DPFC_DISABLE_DUMMY0);
 }
@@ -386,13 +386,53 @@ static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enabl
 	return was_enabled;
 }
 
+/**
+ * intel_set_memory_cxsr - Configure CxSR state
+ * @dev_priv: i915 device
+ * @enable: Allow vs. disallow CxSR
+ *
+ * Allow or disallow the system to enter a special CxSR
+ * (C-state self refresh) state. What typically happens in CxSR mode
+ * is that several display FIFOs may get combined into a single larger
+ * FIFO for a particular plane (so called max FIFO mode) to allow the
+ * system to defer memory fetches longer, and the memory will enter
+ * self refresh.
+ *
+ * Note that enabling CxSR does not guarantee that the system enter
+ * this special mode, nor does it guarantee that the system stays
+ * in that mode once entered. So this just allows/disallows the system
+ * to autonomously utilize the CxSR mode. Other factors such as core
+ * C-states will affect when/if the system actually enters/exits the
+ * CxSR mode.
+ *
+ * Note that on VLV/CHV this actually only controls the max FIFO mode,
+ * and the system is free to enter/exit memory self refresh at any time
+ * even when the use of CxSR has been disallowed.
+ *
+ * While the system is actually in the CxSR/max FIFO mode, some plane
+ * control registers will not get latched on vblank. Thus in order to
+ * guarantee the system will respond to changes in the plane registers
+ * we must always disallow CxSR prior to making changes to those registers.
+ * Unfortunately the system will re-evaluate the CxSR conditions at
+ * frame start which happens after vblank start (which is when the plane
+ * registers would get latched), so we can't proceed with the plane update
+ * during the same frame where we disallowed CxSR.
+ *
+ * Certain platforms also have a deeper HPLL SR mode. Fortunately the
+ * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
+ * the hardware w.r.t. HPLL SR when writing to plane registers.
+ * Disallowing just CxSR is sufficient.
+ */
 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 {
 	bool ret;
 
 	mutex_lock(&dev_priv->wm.wm_mutex);
 	ret = _intel_set_memory_cxsr(dev_priv, enable);
-	dev_priv->wm.vlv.cxsr = enable;
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		dev_priv->wm.vlv.cxsr = enable;
+	else if (IS_G4X(dev_priv))
+		dev_priv->wm.g4x.cxsr = enable;
 	mutex_unlock(&dev_priv->wm.wm_mutex);
 
 	return ret;
@@ -454,13 +494,6 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 	fifo_state->plane[PLANE_CURSOR] = 63;
-
-	DRM_DEBUG_KMS("Pipe %c FIFO size: %d/%d/%d/%d\n",
-		      pipe_name(pipe),
-		      fifo_state->plane[PLANE_PRIMARY],
-		      fifo_state->plane[PLANE_SPRITE0],
-		      fifo_state->plane[PLANE_SPRITE1],
-		      fifo_state->plane[PLANE_CURSOR]);
 }
 
 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
@@ -538,20 +571,6 @@ static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
 	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 };
-static const struct intel_watermark_params g4x_wm_info = {
-	.fifo_size = G4X_FIFO_SIZE,
-	.max_wm = G4X_MAX_WM,
-	.default_wm = G4X_MAX_WM,
-	.guard_size = 2,
-	.cacheline_size = G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params g4x_cursor_wm_info = {
-	.fifo_size = I965_CURSOR_FIFO,
-	.max_wm = I965_CURSOR_MAX_WM,
-	.default_wm = I965_CURSOR_DFT_WM,
-	.guard_size = 2,
-	.cacheline_size = G4X_FIFO_LINE_SIZE,
-};
 static const struct intel_watermark_params i965_cursor_wm_info = {
 	.fifo_size = I965_CURSOR_FIFO,
 	.max_wm = I965_CURSOR_MAX_WM,
@@ -596,8 +615,104 @@ static const struct intel_watermark_params i845_wm_info = {
 };
 
 /**
+ * intel_wm_method1 - Method 1 / "small buffer" watermark formula
+ * @pixel_rate: Pipe pixel rate in kHz
+ * @cpp: Plane bytes per pixel
+ * @latency: Memory wakeup latency in 0.1us units
+ *
+ * Compute the watermark using the method 1 or "small buffer"
+ * formula. The caller may additonally add extra cachelines
+ * to account for TLB misses and clock crossings.
+ *
+ * This method is concerned with the short term drain rate
+ * of the FIFO, ie. it does not account for blanking periods
+ * which would effectively reduce the average drain rate across
+ * a longer period. The name "small" refers to the fact the
+ * FIFO is relatively small compared to the amount of data
+ * fetched.
+ *
+ * The FIFO level vs. time graph might look something like:
+ *
+ *   |\   |\
+ *   | \  | \
+ * __---__---__ (- plane active, _ blanking)
+ * -> time
+ *
+ * or perhaps like this:
+ *
+ *   |\|\  |\|\
+ * __----__----__ (- plane active, _ blanking)
+ * -> time
+ *
+ * Returns:
+ * The watermark in bytes
+ */
+static unsigned int intel_wm_method1(unsigned int pixel_rate,
+				     unsigned int cpp,
+				     unsigned int latency)
+{
+	uint64_t ret;
+
+	ret = (uint64_t) pixel_rate * cpp * latency;
+	ret = DIV_ROUND_UP_ULL(ret, 10000);
+
+	return ret;
+}
+
+/**
+ * intel_wm_method2 - Method 2 / "large buffer" watermark formula
+ * @pixel_rate: Pipe pixel rate in kHz
+ * @htotal: Pipe horizontal total
+ * @width: Plane width in pixels
+ * @cpp: Plane bytes per pixel
+ * @latency: Memory wakeup latency in 0.1us units
+ *
+ * Compute the watermark using the method 2 or "large buffer"
+ * formula. The caller may additonally add extra cachelines
+ * to account for TLB misses and clock crossings.
+ *
+ * This method is concerned with the long term drain rate
+ * of the FIFO, ie. it does account for blanking periods
+ * which effectively reduce the average drain rate across
+ * a longer period. The name "large" refers to the fact the
+ * FIFO is relatively large compared to the amount of data
+ * fetched.
+ *
+ * The FIFO level vs. time graph might look something like:
+ *
+ *    |\___       |\___
+ *    |    \___   |    \___
+ *    |        \  |        \
+ * __ --__--__--__--__--__--__ (- plane active, _ blanking)
+ * -> time
+ *
+ * Returns:
+ * The watermark in bytes
+ */
+static unsigned int intel_wm_method2(unsigned int pixel_rate,
+				     unsigned int htotal,
+				     unsigned int width,
+				     unsigned int cpp,
+				     unsigned int latency)
+{
+	unsigned int ret;
+
+	/*
+	 * FIXME remove once all users are computing
+	 * watermarks in the correct place.
+	 */
+	if (WARN_ON_ONCE(htotal == 0))
+		htotal = 1;
+
+	ret = (latency * pixel_rate) / (htotal * 10000);
+	ret = (ret + 1) * width * cpp;
+
+	return ret;
+}
+
+/**
  * intel_calculate_wm - calculate watermark level
- * @clock_in_khz: pixel clock
+ * @pixel_rate: pixel clock
  * @wm: chip FIFO params
  * @cpp: bytes per pixel
  * @latency_ns: memory latency for the platform
@@ -613,12 +728,12 @@ static const struct intel_watermark_params i845_wm_info = {
  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
  * will occur, and a display engine hang could result.
  */
-static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
-					const struct intel_watermark_params *wm,
-					int fifo_size, int cpp,
-					unsigned long latency_ns)
+static unsigned int intel_calculate_wm(int pixel_rate,
+				       const struct intel_watermark_params *wm,
+				       int fifo_size, int cpp,
+				       unsigned int latency_ns)
 {
-	long entries_required, wm_size;
+	int entries, wm_size;
 
 	/*
 	 * Note: we need to make sure we don't overflow for various clock &
@@ -626,18 +741,17 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 	 * clocks go from a few thousand to several hundred thousand.
 	 * latency is usually a few thousand
 	 */
-	entries_required = ((clock_in_khz / 1000) * cpp * latency_ns) /
-		1000;
-	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
-
-	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
+	entries = intel_wm_method1(pixel_rate, cpp,
+				   latency_ns / 100);
+	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
+		wm->guard_size;
+	DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 
-	wm_size = fifo_size - (entries_required + wm->guard_size);
-
-	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
+	wm_size = fifo_size - entries;
+	DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 
 	/* Don't promote wm_size to unsigned... */
-	if (wm_size > (long)wm->max_wm)
+	if (wm_size > wm->max_wm)
 		wm_size = wm->max_wm;
 	if (wm_size <= 0)
 		wm_size = wm->default_wm;
@@ -655,6 +769,21 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 	return wm_size;
 }
 
+static bool is_disabling(int old, int new, int threshold)
+{
+	return old >= threshold && new < threshold;
+}
+
+static bool is_enabling(int old, int new, int threshold)
+{
+	return old < threshold && new >= threshold;
+}
+
+static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
+{
+	return dev_priv->wm.max_level + 1;
+}
+
 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 				   const struct intel_plane_state *plane_state)
 {
@@ -699,7 +828,7 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
 	struct intel_crtc *crtc;
 	const struct cxsr_latency *latency;
 	u32 reg;
-	unsigned long wm;
+	unsigned int wm;
 
 	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 					 dev_priv->is_ddr3,
@@ -733,7 +862,7 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
 		/* cursor SR */
 		wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 					pineview_display_wm.fifo_size,
-					cpp, latency->cursor_sr);
+					4, latency->cursor_sr);
 		reg = I915_READ(DSPFW3);
 		reg &= ~DSPFW_CURSOR_SR_MASK;
 		reg |= FW_WM(wm, CURSOR_SR);
@@ -751,7 +880,7 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
 		/* cursor HPLL off SR */
 		wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 					pineview_display_hplloff_wm.fifo_size,
-					cpp, latency->cursor_hpll_disable);
+					4, latency->cursor_hpll_disable);
 		reg = I915_READ(DSPFW3);
 		reg &= ~DSPFW_HPLL_CURSOR_MASK;
 		reg |= FW_WM(wm, HPLL_CURSOR);
@@ -764,144 +893,50 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
 	}
 }
 
-static bool g4x_compute_wm0(struct drm_i915_private *dev_priv,
-			    int plane,
-			    const struct intel_watermark_params *display,
-			    int display_latency_ns,
-			    const struct intel_watermark_params *cursor,
-			    int cursor_latency_ns,
-			    int *plane_wm,
-			    int *cursor_wm)
-{
-	struct intel_crtc *crtc;
-	const struct drm_display_mode *adjusted_mode;
-	const struct drm_framebuffer *fb;
-	int htotal, hdisplay, clock, cpp;
-	int line_time_us, line_count;
-	int entries, tlb_miss;
-
-	crtc = intel_get_crtc_for_plane(dev_priv, plane);
-	if (!intel_crtc_active(crtc)) {
-		*cursor_wm = cursor->guard_size;
-		*plane_wm = display->guard_size;
-		return false;
-	}
-
-	adjusted_mode = &crtc->config->base.adjusted_mode;
-	fb = crtc->base.primary->state->fb;
-	clock = adjusted_mode->crtc_clock;
-	htotal = adjusted_mode->crtc_htotal;
-	hdisplay = crtc->config->pipe_src_w;
-	cpp = fb->format->cpp[0];
-
-	/* Use the small buffer method to calculate plane watermark */
-	entries = ((clock * cpp / 1000) * display_latency_ns) / 1000;
-	tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
-	if (tlb_miss > 0)
-		entries += tlb_miss;
-	entries = DIV_ROUND_UP(entries, display->cacheline_size);
-	*plane_wm = entries + display->guard_size;
-	if (*plane_wm > (int)display->max_wm)
-		*plane_wm = display->max_wm;
-
-	/* Use the large buffer method to calculate cursor watermark */
-	line_time_us = max(htotal * 1000 / clock, 1);
-	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
-	entries = line_count * crtc->base.cursor->state->crtc_w * cpp;
-	tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
-	if (tlb_miss > 0)
-		entries += tlb_miss;
-	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
-	*cursor_wm = entries + cursor->guard_size;
-	if (*cursor_wm > (int)cursor->max_wm)
-		*cursor_wm = (int)cursor->max_wm;
-
-	return true;
-}
-
 /*
- * Check the wm result.
- *
- * If any calculated watermark values is larger than the maximum value that
- * can be programmed into the associated watermark register, that watermark
- * must be disabled.
+ * Documentation says:
+ * "If the line size is small, the TLB fetches can get in the way of the
+ *  data fetches, causing some lag in the pixel data return which is not
+ *  accounted for in the above formulas. The following adjustment only
+ *  needs to be applied if eight whole lines fit in the buffer at once.
+ *  The WM is adjusted upwards by the difference between the FIFO size
+ *  and the size of 8 whole lines. This adjustment is always performed
+ *  in the actual pixel depth regardless of whether FBC is enabled or not."
  */
-static bool g4x_check_srwm(struct drm_i915_private *dev_priv,
-			   int display_wm, int cursor_wm,
-			   const struct intel_watermark_params *display,
-			   const struct intel_watermark_params *cursor)
+static int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 {
-	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
-		      display_wm, cursor_wm);
-
-	if (display_wm > display->max_wm) {
-		DRM_DEBUG_KMS("display watermark is too large(%d/%u), disabling\n",
-			      display_wm, display->max_wm);
-		return false;
-	}
+	int tlb_miss = fifo_size * 64 - width * cpp * 8;
 
-	if (cursor_wm > cursor->max_wm) {
-		DRM_DEBUG_KMS("cursor watermark is too large(%d/%u), disabling\n",
-			      cursor_wm, cursor->max_wm);
-		return false;
-	}
-
-	if (!(display_wm || cursor_wm)) {
-		DRM_DEBUG_KMS("SR latency is 0, disabling\n");
-		return false;
-	}
-
-	return true;
+	return max(0, tlb_miss);
 }
 
-static bool g4x_compute_srwm(struct drm_i915_private *dev_priv,
-			     int plane,
-			     int latency_ns,
-			     const struct intel_watermark_params *display,
-			     const struct intel_watermark_params *cursor,
-			     int *display_wm, int *cursor_wm)
+static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
+				const struct g4x_wm_values *wm)
 {
-	struct intel_crtc *crtc;
-	const struct drm_display_mode *adjusted_mode;
-	const struct drm_framebuffer *fb;
-	int hdisplay, htotal, cpp, clock;
-	unsigned long line_time_us;
-	int line_count, line_size;
-	int small, large;
-	int entries;
-
-	if (!latency_ns) {
-		*display_wm = *cursor_wm = 0;
-		return false;
-	}
-
-	crtc = intel_get_crtc_for_plane(dev_priv, plane);
-	adjusted_mode = &crtc->config->base.adjusted_mode;
-	fb = crtc->base.primary->state->fb;
-	clock = adjusted_mode->crtc_clock;
-	htotal = adjusted_mode->crtc_htotal;
-	hdisplay = crtc->config->pipe_src_w;
-	cpp = fb->format->cpp[0];
-
-	line_time_us = max(htotal * 1000 / clock, 1);
-	line_count = (latency_ns / line_time_us + 1000) / 1000;
-	line_size = hdisplay * cpp;
-
-	/* Use the minimum of the small and large buffer method for primary */
-	small = ((clock * cpp / 1000) * latency_ns) / 1000;
-	large = line_count * line_size;
+	enum pipe pipe;
 
-	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
-	*display_wm = entries + display->guard_size;
+	for_each_pipe(dev_priv, pipe)
+		trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 
-	/* calculate the self-refresh watermark for display cursor */
-	entries = line_count * cpp * crtc->base.cursor->state->crtc_w;
-	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
-	*cursor_wm = entries + cursor->guard_size;
+	I915_WRITE(DSPFW1,
+		   FW_WM(wm->sr.plane, SR) |
+		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
+		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
+		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
+	I915_WRITE(DSPFW2,
+		   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
+		   FW_WM(wm->sr.fbc, FBC_SR) |
+		   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
+		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
+		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
+		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
+	I915_WRITE(DSPFW3,
+		   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
+		   FW_WM(wm->sr.cursor, CURSOR_SR) |
+		   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
+		   FW_WM(wm->hpll.plane, HPLL_SR));
 
-	return g4x_check_srwm(dev_priv,
-			      *display_wm, *cursor_wm,
-			      display, cursor);
+	POSTING_READ(DSPFW1);
 }
 
 #define FW_WM_VLV(value, plane) \
@@ -985,17 +1020,535 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 
 #undef FW_WM_VLV
 
+static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
+{
+	/* all latencies in usec */
+	dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
+	dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
+	dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
+
+	dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
+}
+
+static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
+{
+	/*
+	 * DSPCNTR[13] supposedly controls whether the
+	 * primary plane can use the FIFO space otherwise
+	 * reserved for the sprite plane. It's not 100% clear
+	 * what the actual FIFO size is, but it looks like we
+	 * can happily set both primary and sprite watermarks
+	 * up to 127 cachelines. So that would seem to mean
+	 * that either DSPCNTR[13] doesn't do anything, or that
+	 * the total FIFO is >= 256 cachelines in size. Either
+	 * way, we don't seem to have to worry about this
+	 * repartitioning as the maximum watermark value the
+	 * register can hold for each plane is lower than the
+	 * minimum FIFO size.
+	 */
+	switch (plane_id) {
+	case PLANE_CURSOR:
+		return 63;
+	case PLANE_PRIMARY:
+		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
+	case PLANE_SPRITE0:
+		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
+	default:
+		MISSING_CASE(plane_id);
+		return 0;
+	}
+}
+
+static int g4x_fbc_fifo_size(int level)
+{
+	switch (level) {
+	case G4X_WM_LEVEL_SR:
+		return 7;
+	case G4X_WM_LEVEL_HPLL:
+		return 15;
+	default:
+		MISSING_CASE(level);
+		return 0;
+	}
+}
+
+static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
+			       const struct intel_plane_state *plane_state,
+			       int level)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
+	int clock, htotal, cpp, width, wm;
+	int latency = dev_priv->wm.pri_latency[level] * 10;
+
+	if (latency == 0)
+		return USHRT_MAX;
+
+	if (!intel_wm_plane_visible(crtc_state, plane_state))
+		return 0;
+
+	/*
+	 * Not 100% sure which way ELK should go here as the
+	 * spec only says CL/CTG should assume 32bpp and BW
+	 * doesn't need to. But as these things followed the
+	 * mobile vs. desktop lines on gen3 as well, let's
+	 * assume ELK doesn't need this.
+	 *
+	 * The spec also fails to list such a restriction for
+	 * the HPLL watermark, which seems a little strange.
+	 * Let's use 32bpp for the HPLL watermark as well.
+	 */
+	if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
+	    level != G4X_WM_LEVEL_NORMAL)
+		cpp = 4;
+	else
+		cpp = plane_state->base.fb->format->cpp[0];
+
+	clock = adjusted_mode->crtc_clock;
+	htotal = adjusted_mode->crtc_htotal;
+
+	if (plane->id == PLANE_CURSOR)
+		width = plane_state->base.crtc_w;
+	else
+		width = drm_rect_width(&plane_state->base.dst);
+
+	if (plane->id == PLANE_CURSOR) {
+		wm = intel_wm_method2(clock, htotal, width, cpp, latency);
+	} else if (plane->id == PLANE_PRIMARY &&
+		   level == G4X_WM_LEVEL_NORMAL) {
+		wm = intel_wm_method1(clock, cpp, latency);
+	} else {
+		int small, large;
+
+		small = intel_wm_method1(clock, cpp, latency);
+		large = intel_wm_method2(clock, htotal, width, cpp, latency);
+
+		wm = min(small, large);
+	}
+
+	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
+			      width, cpp);
+
+	wm = DIV_ROUND_UP(wm, 64) + 2;
+
+	return min_t(int, wm, USHRT_MAX);
+}
+
+static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
+				 int level, enum plane_id plane_id, u16 value)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	bool dirty = false;
+
+	for (; level < intel_wm_num_levels(dev_priv); level++) {
+		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
+
+		dirty |= raw->plane[plane_id] != value;
+		raw->plane[plane_id] = value;
+	}
+
+	return dirty;
+}
+
+static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
+			       int level, u16 value)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	bool dirty = false;
+
+	/* NORMAL level doesn't have an FBC watermark */
+	level = max(level, G4X_WM_LEVEL_SR);
+
+	for (; level < intel_wm_num_levels(dev_priv); level++) {
+		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
+
+		dirty |= raw->fbc != value;
+		raw->fbc = value;
+	}
+
+	return dirty;
+}
+
+static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
+				   const struct intel_plane_state *pstate,
+				   uint32_t pri_val);
+
+static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
+				     const struct intel_plane_state *plane_state)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
+	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
+	enum plane_id plane_id = plane->id;
+	bool dirty = false;
+	int level;
+
+	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
+		dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
+		if (plane_id == PLANE_PRIMARY)
+			dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
+		goto out;
+	}
+
+	for (level = 0; level < num_levels; level++) {
+		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
+		int wm, max_wm;
+
+		wm = g4x_compute_wm(crtc_state, plane_state, level);
+		max_wm = g4x_plane_fifo_size(plane_id, level);
+
+		if (wm > max_wm)
+			break;
+
+		dirty |= raw->plane[plane_id] != wm;
+		raw->plane[plane_id] = wm;
+
+		if (plane_id != PLANE_PRIMARY ||
+		    level == G4X_WM_LEVEL_NORMAL)
+			continue;
+
+		wm = ilk_compute_fbc_wm(crtc_state, plane_state,
+					raw->plane[plane_id]);
+		max_wm = g4x_fbc_fifo_size(level);
+
+		/*
+		 * FBC wm is not mandatory as we
+		 * can always just disable its use.
+		 */
+		if (wm > max_wm)
+			wm = USHRT_MAX;
+
+		dirty |= raw->fbc != wm;
+		raw->fbc = wm;
+	}
+
+	/* mark watermarks as invalid */
+	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
+
+	if (plane_id == PLANE_PRIMARY)
+		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
+
+ out:
+	if (dirty) {
+		DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
+			      plane->base.name,
+			      crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
+			      crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
+			      crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
+
+		if (plane_id == PLANE_PRIMARY)
+			DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
+				      crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
+				      crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
+	}
+
+	return dirty;
+}
+
+static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
+				      enum plane_id plane_id, int level)
+{
+	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
+
+	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
+}
+
+static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
+				     int level)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+
+	if (level > dev_priv->wm.max_level)
+		return false;
+
+	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
+		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
+		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
+}
+
+/* mark all levels starting from 'level' as invalid */
+static void g4x_invalidate_wms(struct intel_crtc *crtc,
+			       struct g4x_wm_state *wm_state, int level)
+{
+	if (level <= G4X_WM_LEVEL_NORMAL) {
+		enum plane_id plane_id;
+
+		for_each_plane_id_on_crtc(crtc, plane_id)
+			wm_state->wm.plane[plane_id] = USHRT_MAX;
+	}
+
+	if (level <= G4X_WM_LEVEL_SR) {
+		wm_state->cxsr = false;
+		wm_state->sr.cursor = USHRT_MAX;
+		wm_state->sr.plane = USHRT_MAX;
+		wm_state->sr.fbc = USHRT_MAX;
+	}
+
+	if (level <= G4X_WM_LEVEL_HPLL) {
+		wm_state->hpll_en = false;
+		wm_state->hpll.cursor = USHRT_MAX;
+		wm_state->hpll.plane = USHRT_MAX;
+		wm_state->hpll.fbc = USHRT_MAX;
+	}
+}
+
+static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct intel_atomic_state *state =
+		to_intel_atomic_state(crtc_state->base.state);
+	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
+	int num_active_planes = hweight32(crtc_state->active_planes &
+					  ~BIT(PLANE_CURSOR));
+	const struct g4x_pipe_wm *raw;
+	struct intel_plane_state *plane_state;
+	struct intel_plane *plane;
+	enum plane_id plane_id;
+	int i, level;
+	unsigned int dirty = 0;
+
+	for_each_intel_plane_in_state(state, plane, plane_state, i) {
+		const struct intel_plane_state *old_plane_state =
+			to_intel_plane_state(plane->base.state);
+
+		if (plane_state->base.crtc != &crtc->base &&
+		    old_plane_state->base.crtc != &crtc->base)
+			continue;
+
+		if (g4x_raw_plane_wm_compute(crtc_state, plane_state))
+			dirty |= BIT(plane->id);
+	}
+
+	if (!dirty)
+		return 0;
+
+	level = G4X_WM_LEVEL_NORMAL;
+	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
+		goto out;
+
+	raw = &crtc_state->wm.g4x.raw[level];
+	for_each_plane_id_on_crtc(crtc, plane_id)
+		wm_state->wm.plane[plane_id] = raw->plane[plane_id];
+
+	level = G4X_WM_LEVEL_SR;
+
+	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
+		goto out;
+
+	raw = &crtc_state->wm.g4x.raw[level];
+	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
+	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
+	wm_state->sr.fbc = raw->fbc;
+
+	wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
+
+	level = G4X_WM_LEVEL_HPLL;
+
+	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
+		goto out;
+
+	raw = &crtc_state->wm.g4x.raw[level];
+	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
+	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
+	wm_state->hpll.fbc = raw->fbc;
+
+	wm_state->hpll_en = wm_state->cxsr;
+
+	level++;
+
+ out:
+	if (level == G4X_WM_LEVEL_NORMAL)
+		return -EINVAL;
+
+	/* invalidate the higher levels */
+	g4x_invalidate_wms(crtc, wm_state, level);
+
+	/*
+	 * Determine if the FBC watermark(s) can be used. IF
+	 * this isn't the case we prefer to disable the FBC
+	 ( watermark(s) rather than disable the SR/HPLL
+	 * level(s) entirely.
+	 */
+	wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
+
+	if (level >= G4X_WM_LEVEL_SR &&
+	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
+		wm_state->fbc_en = false;
+	else if (level >= G4X_WM_LEVEL_HPLL &&
+		 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
+		wm_state->fbc_en = false;
+
+	return 0;
+}
+
+static int g4x_compute_intermediate_wm(struct drm_device *dev,
+				       struct intel_crtc *crtc,
+				       struct intel_crtc_state *crtc_state)
+{
+	struct g4x_wm_state *intermediate = &crtc_state->wm.g4x.intermediate;
+	const struct g4x_wm_state *optimal = &crtc_state->wm.g4x.optimal;
+	const struct g4x_wm_state *active = &crtc->wm.active.g4x;
+	enum plane_id plane_id;
+
+	intermediate->cxsr = optimal->cxsr && active->cxsr &&
+		!crtc_state->disable_cxsr;
+	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
+		!crtc_state->disable_cxsr;
+	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		intermediate->wm.plane[plane_id] =
+			max(optimal->wm.plane[plane_id],
+			    active->wm.plane[plane_id]);
+
+		WARN_ON(intermediate->wm.plane[plane_id] >
+			g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
+	}
+
+	intermediate->sr.plane = max(optimal->sr.plane,
+				     active->sr.plane);
+	intermediate->sr.cursor = max(optimal->sr.cursor,
+				      active->sr.cursor);
+	intermediate->sr.fbc = max(optimal->sr.fbc,
+				   active->sr.fbc);
+
+	intermediate->hpll.plane = max(optimal->hpll.plane,
+				       active->hpll.plane);
+	intermediate->hpll.cursor = max(optimal->hpll.cursor,
+					active->hpll.cursor);
+	intermediate->hpll.fbc = max(optimal->hpll.fbc,
+				     active->hpll.fbc);
+
+	WARN_ON((intermediate->sr.plane >
+		 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
+		 intermediate->sr.cursor >
+		 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
+		intermediate->cxsr);
+	WARN_ON((intermediate->sr.plane >
+		 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
+		 intermediate->sr.cursor >
+		 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
+		intermediate->hpll_en);
+
+	WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
+		intermediate->fbc_en && intermediate->cxsr);
+	WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
+		intermediate->fbc_en && intermediate->hpll_en);
+
+	/*
+	 * If our intermediate WM are identical to the final WM, then we can
+	 * omit the post-vblank programming; only update if it's different.
+	 */
+	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
+		crtc_state->wm.need_postvbl_update = true;
+
+	return 0;
+}
+
+static void g4x_merge_wm(struct drm_i915_private *dev_priv,
+			 struct g4x_wm_values *wm)
+{
+	struct intel_crtc *crtc;
+	int num_active_crtcs = 0;
+
+	wm->cxsr = true;
+	wm->hpll_en = true;
+	wm->fbc_en = true;
+
+	for_each_intel_crtc(&dev_priv->drm, crtc) {
+		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
+
+		if (!crtc->active)
+			continue;
+
+		if (!wm_state->cxsr)
+			wm->cxsr = false;
+		if (!wm_state->hpll_en)
+			wm->hpll_en = false;
+		if (!wm_state->fbc_en)
+			wm->fbc_en = false;
+
+		num_active_crtcs++;
+	}
+
+	if (num_active_crtcs != 1) {
+		wm->cxsr = false;
+		wm->hpll_en = false;
+		wm->fbc_en = false;
+	}
+
+	for_each_intel_crtc(&dev_priv->drm, crtc) {
+		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
+		enum pipe pipe = crtc->pipe;
+
+		wm->pipe[pipe] = wm_state->wm;
+		if (crtc->active && wm->cxsr)
+			wm->sr = wm_state->sr;
+		if (crtc->active && wm->hpll_en)
+			wm->hpll = wm_state->hpll;
+	}
+}
+
+static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
+{
+	struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
+	struct g4x_wm_values new_wm = {};
+
+	g4x_merge_wm(dev_priv, &new_wm);
+
+	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
+		return;
+
+	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
+		_intel_set_memory_cxsr(dev_priv, false);
+
+	g4x_write_wm_values(dev_priv, &new_wm);
+
+	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
+		_intel_set_memory_cxsr(dev_priv, true);
+
+	*old_wm = new_wm;
+}
+
+static void g4x_initial_watermarks(struct intel_atomic_state *state,
+				   struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+
+	mutex_lock(&dev_priv->wm.wm_mutex);
+	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
+	g4x_program_watermarks(dev_priv);
+	mutex_unlock(&dev_priv->wm.wm_mutex);
+}
+
+static void g4x_optimize_watermarks(struct intel_atomic_state *state,
+				    struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
+
+	if (!crtc_state->wm.need_postvbl_update)
+		return;
+
+	mutex_lock(&dev_priv->wm.wm_mutex);
+	intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
+	g4x_program_watermarks(dev_priv);
+	mutex_unlock(&dev_priv->wm.wm_mutex);
+}
+
 /* latency must be in 0.1us units. */
 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
-				   unsigned int pipe_htotal,
-				   unsigned int horiz_pixels,
+				   unsigned int htotal,
+				   unsigned int width,
 				   unsigned int cpp,
 				   unsigned int latency)
 {
 	unsigned int ret;
 
-	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
-	ret = (ret + 1) * horiz_pixels * cpp;
+	ret = intel_wm_method2(pixel_rate, htotal,
+			       width, cpp, latency);
 	ret = DIV_ROUND_UP(ret, 64);
 
 	return ret;
@@ -1029,17 +1582,15 @@ static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
 	if (dev_priv->wm.pri_latency[level] == 0)
 		return USHRT_MAX;
 
-	if (!plane_state->base.visible)
+	if (!intel_wm_plane_visible(crtc_state, plane_state))
 		return 0;
 
 	cpp = plane_state->base.fb->format->cpp[0];
 	clock = adjusted_mode->crtc_clock;
 	htotal = adjusted_mode->crtc_htotal;
 	width = crtc_state->pipe_src_w;
-	if (WARN_ON(htotal == 0))
-		htotal = 1;
 
-	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
+	if (plane->id == PLANE_CURSOR) {
 		/*
 		 * FIXME the formula gives values that are
 		 * too big for the cursor FIFO, and hence we
@@ -1064,7 +1615,7 @@ static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
-	const struct vlv_pipe_wm *raw =
+	const struct g4x_pipe_wm *raw =
 		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
 	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 	unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
@@ -1143,18 +1694,13 @@ static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
 	return 0;
 }
 
-static int vlv_num_wm_levels(struct drm_i915_private *dev_priv)
-{
-	return dev_priv->wm.max_level + 1;
-}
-
 /* mark all levels starting from 'level' as invalid */
 static void vlv_invalidate_wms(struct intel_crtc *crtc,
 			       struct vlv_wm_state *wm_state, int level)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	for (; level < vlv_num_wm_levels(dev_priv); level++) {
+	for (; level < intel_wm_num_levels(dev_priv); level++) {
 		enum plane_id plane_id;
 
 		for_each_plane_id_on_crtc(crtc, plane_id)
@@ -1181,11 +1727,11 @@ static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
 				 int level, enum plane_id plane_id, u16 value)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
-	int num_levels = vlv_num_wm_levels(dev_priv);
+	int num_levels = intel_wm_num_levels(dev_priv);
 	bool dirty = false;
 
 	for (; level < num_levels; level++) {
-		struct vlv_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
+		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
 
 		dirty |= raw->plane[plane_id] != value;
 		raw->plane[plane_id] = value;
@@ -1194,22 +1740,22 @@ static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
 	return dirty;
 }
 
-static bool vlv_plane_wm_compute(struct intel_crtc_state *crtc_state,
-				 const struct intel_plane_state *plane_state)
+static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
+				     const struct intel_plane_state *plane_state)
 {
 	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 	enum plane_id plane_id = plane->id;
-	int num_levels = vlv_num_wm_levels(to_i915(plane->base.dev));
+	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
 	int level;
 	bool dirty = false;
 
-	if (!plane_state->base.visible) {
+	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
 		dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
 		goto out;
 	}
 
 	for (level = 0; level < num_levels; level++) {
-		struct vlv_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
+		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
 		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
 		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
 
@@ -1225,7 +1771,7 @@ static bool vlv_plane_wm_compute(struct intel_crtc_state *crtc_state,
 
 out:
 	if (dirty)
-		DRM_DEBUG_KMS("%s wms: [0]=%d,[1]=%d,[2]=%d\n",
+		DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
 			      plane->base.name,
 			      crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
 			      crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
@@ -1234,10 +1780,10 @@ out:
 	return dirty;
 }
 
-static bool vlv_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
-				  enum plane_id plane_id, int level)
+static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
+				      enum plane_id plane_id, int level)
 {
-	const struct vlv_pipe_wm *raw =
+	const struct g4x_pipe_wm *raw =
 		&crtc_state->wm.vlv.raw[level];
 	const struct vlv_fifo_state *fifo_state =
 		&crtc_state->wm.vlv.fifo_state;
@@ -1245,12 +1791,12 @@ static bool vlv_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
 	return raw->plane[plane_id] <= fifo_state->plane[plane_id];
 }
 
-static bool vlv_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
+static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
 {
-	return vlv_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
-		vlv_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
-		vlv_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
-		vlv_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
+	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
+		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
+		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
+		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
 }
 
 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
@@ -1279,7 +1825,7 @@ static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
 		    old_plane_state->base.crtc != &crtc->base)
 			continue;
 
-		if (vlv_plane_wm_compute(crtc_state, plane_state))
+		if (vlv_raw_plane_wm_compute(crtc_state, plane_state))
 			dirty |= BIT(plane->id);
 	}
 
@@ -1313,7 +1859,7 @@ static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
 	}
 
 	/* initially allow all levels */
-	wm_state->num_levels = vlv_num_wm_levels(dev_priv);
+	wm_state->num_levels = intel_wm_num_levels(dev_priv);
 	/*
 	 * Note that enabling cxsr with no primary/sprite planes
 	 * enabled can wedge the pipe. Hence we only allow cxsr
@@ -1322,10 +1868,10 @@ static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
 	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
 
 	for (level = 0; level < wm_state->num_levels; level++) {
-		const struct vlv_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
+		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
 		const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
 
-		if (!vlv_crtc_wm_is_valid(crtc_state, level))
+		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
 			break;
 
 		for_each_plane_id_on_crtc(crtc, plane_id) {
@@ -1539,16 +2085,6 @@ static void vlv_merge_wm(struct drm_i915_private *dev_priv,
 	}
 }
 
-static bool is_disabling(int old, int new, int threshold)
-{
-	return old >= threshold && new < threshold;
-}
-
-static bool is_enabling(int old, int new, int threshold)
-{
-	return old < threshold && new >= threshold;
-}
-
 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
 {
 	struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
@@ -1609,65 +2145,6 @@ static void vlv_optimize_watermarks(struct intel_atomic_state *state,
 	mutex_unlock(&dev_priv->wm.wm_mutex);
 }
 
-#define single_plane_enabled(mask) is_power_of_2(mask)
-
-static void g4x_update_wm(struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	static const int sr_latency_ns = 12000;
-	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
-	int plane_sr, cursor_sr;
-	unsigned int enabled = 0;
-	bool cxsr_enabled;
-
-	if (g4x_compute_wm0(dev_priv, PIPE_A,
-			    &g4x_wm_info, pessimal_latency_ns,
-			    &g4x_cursor_wm_info, pessimal_latency_ns,
-			    &planea_wm, &cursora_wm))
-		enabled |= 1 << PIPE_A;
-
-	if (g4x_compute_wm0(dev_priv, PIPE_B,
-			    &g4x_wm_info, pessimal_latency_ns,
-			    &g4x_cursor_wm_info, pessimal_latency_ns,
-			    &planeb_wm, &cursorb_wm))
-		enabled |= 1 << PIPE_B;
-
-	if (single_plane_enabled(enabled) &&
-	    g4x_compute_srwm(dev_priv, ffs(enabled) - 1,
-			     sr_latency_ns,
-			     &g4x_wm_info,
-			     &g4x_cursor_wm_info,
-			     &plane_sr, &cursor_sr)) {
-		cxsr_enabled = true;
-	} else {
-		cxsr_enabled = false;
-		intel_set_memory_cxsr(dev_priv, false);
-		plane_sr = cursor_sr = 0;
-	}
-
-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
-		      "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
-		      planea_wm, cursora_wm,
-		      planeb_wm, cursorb_wm,
-		      plane_sr, cursor_sr);
-
-	I915_WRITE(DSPFW1,
-		   FW_WM(plane_sr, SR) |
-		   FW_WM(cursorb_wm, CURSORB) |
-		   FW_WM(planeb_wm, PLANEB) |
-		   FW_WM(planea_wm, PLANEA));
-	I915_WRITE(DSPFW2,
-		   (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
-		   FW_WM(cursora_wm, CURSORA));
-	/* HPLL off in SR has some issues on G4x... disable it */
-	I915_WRITE(DSPFW3,
-		   (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
-		   FW_WM(cursor_sr, CURSOR_SR));
-
-	if (cxsr_enabled)
-		intel_set_memory_cxsr(dev_priv, true);
-}
-
 static void i965_update_wm(struct intel_crtc *unused_crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
@@ -1689,14 +2166,10 @@ static void i965_update_wm(struct intel_crtc *unused_crtc)
 		int htotal = adjusted_mode->crtc_htotal;
 		int hdisplay = crtc->config->pipe_src_w;
 		int cpp = fb->format->cpp[0];
-		unsigned long line_time_us;
 		int entries;
 
-		line_time_us = max(htotal * 1000 / clock, 1);
-
-		/* Use ns/us then divide to preserve precision */
-		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			cpp * hdisplay;
+		entries = intel_wm_method2(clock, htotal,
+					   hdisplay, cpp, sr_latency_ns / 100);
 		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
 		srwm = I965_FIFO_SIZE - entries;
 		if (srwm < 0)
@@ -1705,13 +2178,14 @@ static void i965_update_wm(struct intel_crtc *unused_crtc)
 		DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
 			      entries, srwm);
 
-		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			cpp * crtc->base.cursor->state->crtc_w;
+		entries = intel_wm_method2(clock, htotal,
+					   crtc->base.cursor->state->crtc_w, 4,
+					   sr_latency_ns / 100);
 		entries = DIV_ROUND_UP(entries,
-					  i965_cursor_wm_info.cacheline_size);
-		cursor_sr = i965_cursor_wm_info.fifo_size -
-			(entries + i965_cursor_wm_info.guard_size);
+				       i965_cursor_wm_info.cacheline_size) +
+			i965_cursor_wm_info.guard_size;
 
+		cursor_sr = i965_cursor_wm_info.fifo_size - entries;
 		if (cursor_sr > i965_cursor_wm_info.max_wm)
 			cursor_sr = i965_cursor_wm_info.max_wm;
 
@@ -1848,7 +2322,6 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc)
 		int htotal = adjusted_mode->crtc_htotal;
 		int hdisplay = enabled->config->pipe_src_w;
 		int cpp;
-		unsigned long line_time_us;
 		int entries;
 
 		if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
@@ -1856,11 +2329,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc)
 		else
 			cpp = fb->format->cpp[0];
 
-		line_time_us = max(htotal * 1000 / clock, 1);
-
-		/* Use ns/us then divide to preserve precision */
-		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			cpp * hdisplay;
+		entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
+					   sr_latency_ns / 100);
 		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
 		DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
 		srwm = wm_info->fifo_size - entries;
@@ -1917,34 +2387,31 @@ static void i845_update_wm(struct intel_crtc *unused_crtc)
 }
 
 /* latency must be in 0.1us units. */
-static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latency)
+static unsigned int ilk_wm_method1(unsigned int pixel_rate,
+				   unsigned int cpp,
+				   unsigned int latency)
 {
-	uint64_t ret;
-
-	if (WARN(latency == 0, "Latency value missing\n"))
-		return UINT_MAX;
+	unsigned int ret;
 
-	ret = (uint64_t) pixel_rate * cpp * latency;
-	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
+	ret = intel_wm_method1(pixel_rate, cpp, latency);
+	ret = DIV_ROUND_UP(ret, 64) + 2;
 
 	return ret;
 }
 
 /* latency must be in 0.1us units. */
-static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
-			       uint32_t horiz_pixels, uint8_t cpp,
-			       uint32_t latency)
+static unsigned int ilk_wm_method2(unsigned int pixel_rate,
+				   unsigned int htotal,
+				   unsigned int width,
+				   unsigned int cpp,
+				   unsigned int latency)
 {
-	uint32_t ret;
-
-	if (WARN(latency == 0, "Latency value missing\n"))
-		return UINT_MAX;
-	if (WARN_ON(!pipe_htotal))
-		return UINT_MAX;
+	unsigned int ret;
 
-	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
-	ret = (ret + 1) * horiz_pixels * cpp;
+	ret = intel_wm_method2(pixel_rate, htotal,
+			       width, cpp, latency);
 	ret = DIV_ROUND_UP(ret, 64) + 2;
+
 	return ret;
 }
 
@@ -3082,7 +3549,7 @@ static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
 static bool
 intel_has_sagv(struct drm_i915_private *dev_priv)
 {
-	if (IS_KABYLAKE(dev_priv))
+	if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
 		return true;
 
 	if (IS_SKYLAKE(dev_priv) &&
@@ -3360,16 +3827,17 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
  * Return value is provided in 16.16 fixed point form to retain fractional part.
  * Caller should take care of dividing & rounding off the value.
  */
-static uint32_t
+static uint_fixed_16_16_t
 skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
 			   const struct intel_plane_state *pstate)
 {
 	struct intel_plane *plane = to_intel_plane(pstate->base.plane);
-	uint32_t downscale_h, downscale_w;
 	uint32_t src_w, src_h, dst_w, dst_h;
+	uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
+	uint_fixed_16_16_t downscale_h, downscale_w;
 
 	if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
-		return DRM_PLANE_HELPER_NO_SCALING;
+		return u32_to_fixed_16_16(0);
 
 	/* n.b., src is 16.16 fixed point, dst is whole integer */
 	if (plane->id == PLANE_CURSOR) {
@@ -3377,8 +3845,8 @@ skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
 		 * Cursors only support 0/180 degree rotation,
 		 * hence no need to account for rotation here.
 		 */
-		src_w = pstate->base.src_w;
-		src_h = pstate->base.src_h;
+		src_w = pstate->base.src_w >> 16;
+		src_h = pstate->base.src_h >> 16;
 		dst_w = pstate->base.crtc_w;
 		dst_h = pstate->base.crtc_h;
 	} else {
@@ -3387,17 +3855,109 @@ skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
 		 * the 90/270 degree plane rotation cases (to match the
 		 * GTT mapping), hence no need to account for rotation here.
 		 */
-		src_w = drm_rect_width(&pstate->base.src);
-		src_h = drm_rect_height(&pstate->base.src);
+		src_w = drm_rect_width(&pstate->base.src) >> 16;
+		src_h = drm_rect_height(&pstate->base.src) >> 16;
 		dst_w = drm_rect_width(&pstate->base.dst);
 		dst_h = drm_rect_height(&pstate->base.dst);
 	}
 
-	downscale_h = max(src_h / dst_h, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
-	downscale_w = max(src_w / dst_w, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
+	fp_w_ratio = fixed_16_16_div(src_w, dst_w);
+	fp_h_ratio = fixed_16_16_div(src_h, dst_h);
+	downscale_w = max_fixed_16_16(fp_w_ratio, u32_to_fixed_16_16(1));
+	downscale_h = max_fixed_16_16(fp_h_ratio, u32_to_fixed_16_16(1));
+
+	return mul_fixed16(downscale_w, downscale_h);
+}
+
+static uint_fixed_16_16_t
+skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
+{
+	uint_fixed_16_16_t pipe_downscale = u32_to_fixed_16_16(1);
+
+	if (!crtc_state->base.enable)
+		return pipe_downscale;
+
+	if (crtc_state->pch_pfit.enabled) {
+		uint32_t src_w, src_h, dst_w, dst_h;
+		uint32_t pfit_size = crtc_state->pch_pfit.size;
+		uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
+		uint_fixed_16_16_t downscale_h, downscale_w;
+
+		src_w = crtc_state->pipe_src_w;
+		src_h = crtc_state->pipe_src_h;
+		dst_w = pfit_size >> 16;
+		dst_h = pfit_size & 0xffff;
+
+		if (!dst_w || !dst_h)
+			return pipe_downscale;
+
+		fp_w_ratio = fixed_16_16_div(src_w, dst_w);
+		fp_h_ratio = fixed_16_16_div(src_h, dst_h);
+		downscale_w = max_fixed_16_16(fp_w_ratio, u32_to_fixed_16_16(1));
+		downscale_h = max_fixed_16_16(fp_h_ratio, u32_to_fixed_16_16(1));
 
-	/* Provide result in 16.16 fixed point */
-	return (uint64_t)downscale_w * downscale_h >> 16;
+		pipe_downscale = mul_fixed16(downscale_w, downscale_h);
+	}
+
+	return pipe_downscale;
+}
+
+int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
+				  struct intel_crtc_state *cstate)
+{
+	struct drm_crtc_state *crtc_state = &cstate->base;
+	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_plane *plane;
+	const struct drm_plane_state *pstate;
+	struct intel_plane_state *intel_pstate;
+	int crtc_clock, dotclk;
+	uint32_t pipe_max_pixel_rate;
+	uint_fixed_16_16_t pipe_downscale;
+	uint_fixed_16_16_t max_downscale = u32_to_fixed_16_16(1);
+
+	if (!cstate->base.enable)
+		return 0;
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		uint_fixed_16_16_t plane_downscale;
+		uint_fixed_16_16_t fp_9_div_8 = fixed_16_16_div(9, 8);
+		int bpp;
+
+		if (!intel_wm_plane_visible(cstate,
+					    to_intel_plane_state(pstate)))
+			continue;
+
+		if (WARN_ON(!pstate->fb))
+			return -EINVAL;
+
+		intel_pstate = to_intel_plane_state(pstate);
+		plane_downscale = skl_plane_downscale_amount(cstate,
+							     intel_pstate);
+		bpp = pstate->fb->format->cpp[0] * 8;
+		if (bpp == 64)
+			plane_downscale = mul_fixed16(plane_downscale,
+						      fp_9_div_8);
+
+		max_downscale = max_fixed_16_16(plane_downscale, max_downscale);
+	}
+	pipe_downscale = skl_pipe_downscale_amount(cstate);
+
+	pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
+
+	crtc_clock = crtc_state->adjusted_mode.crtc_clock;
+	dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
+
+	if (IS_GEMINILAKE(to_i915(intel_crtc->base.dev)))
+		dotclk *= 2;
+
+	pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
+
+	if (pipe_max_pixel_rate < crtc_clock) {
+		DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
+		return -EINVAL;
+	}
+
+	return 0;
 }
 
 static unsigned int
@@ -3407,10 +3967,11 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 {
 	struct intel_plane *plane = to_intel_plane(pstate->plane);
 	struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
-	uint32_t down_scale_amount, data_rate;
+	uint32_t data_rate;
 	uint32_t width = 0, height = 0;
 	struct drm_framebuffer *fb;
 	u32 format;
+	uint_fixed_16_16_t down_scale_amount;
 
 	if (!intel_pstate->base.visible)
 		return 0;
@@ -3446,7 +4007,7 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 
 	down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
 
-	return (uint64_t)data_rate * down_scale_amount >> 16;
+	return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
 }
 
 /*
@@ -3597,6 +4158,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	int num_active;
 	unsigned plane_data_rate[I915_MAX_PLANES] = {};
 	unsigned plane_y_data_rate[I915_MAX_PLANES] = {};
+	uint16_t total_min_blocks = 0;
 
 	/* Clear the partitioning for disabled planes. */
 	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
@@ -3612,10 +4174,8 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 
 	skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active);
 	alloc_size = skl_ddb_entry_size(alloc);
-	if (alloc_size == 0) {
-		memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
+	if (alloc_size == 0)
 		return 0;
-	}
 
 	skl_ddb_calc_min(cstate, num_active, minimum, y_minimum);
 
@@ -3626,10 +4186,18 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	 */
 
 	for_each_plane_id_on_crtc(intel_crtc, plane_id) {
-		alloc_size -= minimum[plane_id];
-		alloc_size -= y_minimum[plane_id];
+		total_min_blocks += minimum[plane_id];
+		total_min_blocks += y_minimum[plane_id];
+	}
+
+	if (total_min_blocks > alloc_size) {
+		DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
+		DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks,
+							alloc_size);
+		return -EINVAL;
 	}
 
+	alloc_size -= total_min_blocks;
 	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
 	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
 
@@ -3708,7 +4276,7 @@ static uint_fixed_16_16_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp,
 		return FP_16_16_MAX;
 
 	wm_intermediate_val = latency * pixel_rate * cpp;
-	ret = fixed_16_16_div_round_up_u64(wm_intermediate_val, 1000 * 512);
+	ret = fixed_16_16_div_u64(wm_intermediate_val, 1000 * 512);
 	return ret;
 }
 
@@ -3730,12 +4298,33 @@ static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
 	return ret;
 }
 
-static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
-					      struct intel_plane_state *pstate)
+static uint_fixed_16_16_t
+intel_get_linetime_us(struct intel_crtc_state *cstate)
+{
+	uint32_t pixel_rate;
+	uint32_t crtc_htotal;
+	uint_fixed_16_16_t linetime_us;
+
+	if (!cstate->base.active)
+		return u32_to_fixed_16_16(0);
+
+	pixel_rate = cstate->pixel_rate;
+
+	if (WARN_ON(pixel_rate == 0))
+		return u32_to_fixed_16_16(0);
+
+	crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
+	linetime_us = fixed_16_16_div_u64(crtc_htotal * 1000, pixel_rate);
+
+	return linetime_us;
+}
+
+static uint32_t
+skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
+			      const struct intel_plane_state *pstate)
 {
 	uint64_t adjusted_pixel_rate;
-	uint64_t downscale_amount;
-	uint64_t pixel_rate;
+	uint_fixed_16_16_t downscale_amount;
 
 	/* Shouldn't reach here on disabled planes... */
 	if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
@@ -3748,15 +4337,13 @@ static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cst
 	adjusted_pixel_rate = cstate->pixel_rate;
 	downscale_amount = skl_plane_downscale_amount(cstate, pstate);
 
-	pixel_rate = adjusted_pixel_rate * downscale_amount >> 16;
-	WARN_ON(pixel_rate != clamp_t(uint32_t, pixel_rate, 0, ~0));
-
-	return pixel_rate;
+	return mul_round_up_u32_fixed16(adjusted_pixel_rate,
+					    downscale_amount);
 }
 
 static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 				struct intel_crtc_state *cstate,
-				struct intel_plane_state *intel_pstate,
+				const struct intel_plane_state *intel_pstate,
 				uint16_t ddb_allocation,
 				int level,
 				uint16_t *out_blocks, /* out */
@@ -3764,8 +4351,8 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 				bool *enabled /* out */)
 {
 	struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
-	struct drm_plane_state *pstate = &intel_pstate->base;
-	struct drm_framebuffer *fb = pstate->fb;
+	const struct drm_plane_state *pstate = &intel_pstate->base;
+	const struct drm_framebuffer *fb = pstate->fb;
 	uint32_t latency = dev_priv->wm.skl_latency[level];
 	uint_fixed_16_16_t method1, method2;
 	uint_fixed_16_16_t plane_blocks_per_line;
@@ -3793,8 +4380,9 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 		  fb->modifier == I915_FORMAT_MOD_Yf_TILED;
 	x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
 
-	/* Display WA #1141: kbl. */
-	if (IS_KABYLAKE(dev_priv) && dev_priv->ipc_enabled)
+	/* Display WA #1141: kbl,cfl */
+	if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
+	    dev_priv->ipc_enabled)
 		latency += 4;
 
 	if (apply_memory_bw_wa && x_tiled)
@@ -3846,8 +4434,8 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	if (y_tiled) {
 		interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line *
 					   y_min_scanlines, 512);
-		plane_blocks_per_line =
-		      fixed_16_16_div_round_up(interm_pbpl, y_min_scanlines);
+		plane_blocks_per_line = fixed_16_16_div(interm_pbpl,
+							y_min_scanlines);
 	} else if (x_tiled) {
 		interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512);
 		plane_blocks_per_line = u32_to_fixed_16_16(interm_pbpl);
@@ -3868,19 +4456,25 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	if (y_tiled) {
 		selected_result = max_fixed_16_16(method2, y_tile_minimum);
 	} else {
+		uint32_t linetime_us;
+
+		linetime_us = fixed_16_16_to_u32_round_up(
+				intel_get_linetime_us(cstate));
 		if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) &&
 		    (plane_bytes_per_line / 512 < 1))
 			selected_result = method2;
-		else if ((ddb_allocation /
+		else if ((ddb_allocation && ddb_allocation /
 			fixed_16_16_to_u32_round_up(plane_blocks_per_line)) >= 1)
 			selected_result = min_fixed_16_16(method1, method2);
+		else if (latency >= linetime_us)
+			selected_result = min_fixed_16_16(method1, method2);
 		else
 			selected_result = method1;
 	}
 
 	res_blocks = fixed_16_16_to_u32_round_up(selected_result) + 1;
-	res_lines = DIV_ROUND_UP(selected_result.val,
-				 plane_blocks_per_line.val);
+	res_lines = div_round_up_fixed16(selected_result,
+					 plane_blocks_per_line);
 
 	if (level >= 1 && level <= 7) {
 		if (y_tiled) {
@@ -3919,54 +4513,39 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 }
 
 static int
-skl_compute_wm_level(const struct drm_i915_private *dev_priv,
-		     struct skl_ddb_allocation *ddb,
-		     struct intel_crtc_state *cstate,
-		     struct intel_plane *intel_plane,
-		     int level,
-		     struct skl_wm_level *result)
+skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
+		      struct skl_ddb_allocation *ddb,
+		      struct intel_crtc_state *cstate,
+		      const struct intel_plane_state *intel_pstate,
+		      struct skl_plane_wm *wm)
 {
-	struct drm_atomic_state *state = cstate->base.state;
 	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
-	struct drm_plane *plane = &intel_plane->base;
-	struct intel_plane_state *intel_pstate = NULL;
+	struct drm_plane *plane = intel_pstate->base.plane;
+	struct intel_plane *intel_plane = to_intel_plane(plane);
 	uint16_t ddb_blocks;
 	enum pipe pipe = intel_crtc->pipe;
+	int level, max_level = ilk_wm_max_level(dev_priv);
 	int ret;
 
-	if (state)
-		intel_pstate =
-			intel_atomic_get_existing_plane_state(state,
-							      intel_plane);
-
-	/*
-	 * Note: If we start supporting multiple pending atomic commits against
-	 * the same planes/CRTC's in the future, plane->state will no longer be
-	 * the correct pre-state to use for the calculations here and we'll
-	 * need to change where we get the 'unchanged' plane data from.
-	 *
-	 * For now this is fine because we only allow one queued commit against
-	 * a CRTC.  Even if the plane isn't modified by this transaction and we
-	 * don't have a plane lock, we still have the CRTC's lock, so we know
-	 * that no other transactions are racing with us to update it.
-	 */
-	if (!intel_pstate)
-		intel_pstate = to_intel_plane_state(plane->state);
-
-	WARN_ON(!intel_pstate->base.fb);
+	if (WARN_ON(!intel_pstate->base.fb))
+		return -EINVAL;
 
 	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][intel_plane->id]);
 
-	ret = skl_compute_plane_wm(dev_priv,
-				   cstate,
-				   intel_pstate,
-				   ddb_blocks,
-				   level,
-				   &result->plane_res_b,
-				   &result->plane_res_l,
-				   &result->plane_en);
-	if (ret)
-		return ret;
+	for (level = 0; level <= max_level; level++) {
+		struct skl_wm_level *result = &wm->wm[level];
+
+		ret = skl_compute_plane_wm(dev_priv,
+					   cstate,
+					   intel_pstate,
+					   ddb_blocks,
+					   level,
+					   &result->plane_res_b,
+					   &result->plane_res_l,
+					   &result->plane_en);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
@@ -3976,19 +4555,16 @@ skl_compute_linetime_wm(struct intel_crtc_state *cstate)
 {
 	struct drm_atomic_state *state = cstate->base.state;
 	struct drm_i915_private *dev_priv = to_i915(state->dev);
-	uint32_t pixel_rate;
+	uint_fixed_16_16_t linetime_us;
 	uint32_t linetime_wm;
 
-	if (!cstate->base.active)
-		return 0;
-
-	pixel_rate = cstate->pixel_rate;
+	linetime_us = intel_get_linetime_us(cstate);
 
-	if (WARN_ON(pixel_rate == 0))
+	if (is_fixed16_zero(linetime_us))
 		return 0;
 
-	linetime_wm = DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal *
-				   1000, pixel_rate);
+	linetime_wm = fixed_16_16_to_u32_round_up(mul_u32_fixed_16_16(8,
+				linetime_us));
 
 	/* Display WA #1135: bxt. */
 	if (IS_BROXTON(dev_priv) && dev_priv->ipc_enabled)
@@ -4012,10 +4588,11 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
 			     struct skl_pipe_wm *pipe_wm)
 {
 	struct drm_device *dev = cstate->base.crtc->dev;
+	struct drm_crtc_state *crtc_state = &cstate->base;
 	const struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane;
+	struct drm_plane *plane;
+	const struct drm_plane_state *pstate;
 	struct skl_plane_wm *wm;
-	int level, max_level = ilk_wm_max_level(dev_priv);
 	int ret;
 
 	/*
@@ -4024,18 +4601,17 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
 	 */
 	memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
 
-	for_each_intel_plane_mask(&dev_priv->drm,
-				  intel_plane,
-				  cstate->base.plane_mask) {
-		wm = &pipe_wm->planes[intel_plane->id];
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		const struct intel_plane_state *intel_pstate =
+						to_intel_plane_state(pstate);
+		enum plane_id plane_id = to_intel_plane(plane)->id;
+
+		wm = &pipe_wm->planes[plane_id];
 
-		for (level = 0; level <= max_level; level++) {
-			ret = skl_compute_wm_level(dev_priv, ddb, cstate,
-						   intel_plane, level,
-						   &wm->wm[level]);
-			if (ret)
-				return ret;
-		}
+		ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
+					    intel_pstate, wm);
+		if (ret)
+			return ret;
 		skl_compute_transition_wm(cstate, &wm->trans_wm);
 	}
 	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
@@ -4675,6 +5251,32 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 #define _FW_WM_VLV(value, plane) \
 	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
 
+static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
+			       struct g4x_wm_values *wm)
+{
+	uint32_t tmp;
+
+	tmp = I915_READ(DSPFW1);
+	wm->sr.plane = _FW_WM(tmp, SR);
+	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
+	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
+	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
+
+	tmp = I915_READ(DSPFW2);
+	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
+	wm->sr.fbc = _FW_WM(tmp, FBC_SR);
+	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
+	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
+	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
+	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
+
+	tmp = I915_READ(DSPFW3);
+	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
+	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
+	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
+	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
+}
+
 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
 			       struct vlv_wm_values *wm)
 {
@@ -4751,6 +5353,147 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
 #undef _FW_WM
 #undef _FW_WM_VLV
 
+void g4x_wm_get_hw_state(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct g4x_wm_values *wm = &dev_priv->wm.g4x;
+	struct intel_crtc *crtc;
+
+	g4x_read_wm_values(dev_priv, wm);
+
+	wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
+
+	for_each_intel_crtc(dev, crtc) {
+		struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+		struct g4x_wm_state *active = &crtc->wm.active.g4x;
+		struct g4x_pipe_wm *raw;
+		enum pipe pipe = crtc->pipe;
+		enum plane_id plane_id;
+		int level, max_level;
+
+		active->cxsr = wm->cxsr;
+		active->hpll_en = wm->hpll_en;
+		active->fbc_en = wm->fbc_en;
+
+		active->sr = wm->sr;
+		active->hpll = wm->hpll;
+
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			active->wm.plane[plane_id] =
+				wm->pipe[pipe].plane[plane_id];
+		}
+
+		if (wm->cxsr && wm->hpll_en)
+			max_level = G4X_WM_LEVEL_HPLL;
+		else if (wm->cxsr)
+			max_level = G4X_WM_LEVEL_SR;
+		else
+			max_level = G4X_WM_LEVEL_NORMAL;
+
+		level = G4X_WM_LEVEL_NORMAL;
+		raw = &crtc_state->wm.g4x.raw[level];
+		for_each_plane_id_on_crtc(crtc, plane_id)
+			raw->plane[plane_id] = active->wm.plane[plane_id];
+
+		if (++level > max_level)
+			goto out;
+
+		raw = &crtc_state->wm.g4x.raw[level];
+		raw->plane[PLANE_PRIMARY] = active->sr.plane;
+		raw->plane[PLANE_CURSOR] = active->sr.cursor;
+		raw->plane[PLANE_SPRITE0] = 0;
+		raw->fbc = active->sr.fbc;
+
+		if (++level > max_level)
+			goto out;
+
+		raw = &crtc_state->wm.g4x.raw[level];
+		raw->plane[PLANE_PRIMARY] = active->hpll.plane;
+		raw->plane[PLANE_CURSOR] = active->hpll.cursor;
+		raw->plane[PLANE_SPRITE0] = 0;
+		raw->fbc = active->hpll.fbc;
+
+	out:
+		for_each_plane_id_on_crtc(crtc, plane_id)
+			g4x_raw_plane_wm_set(crtc_state, level,
+					     plane_id, USHRT_MAX);
+		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
+
+		crtc_state->wm.g4x.optimal = *active;
+		crtc_state->wm.g4x.intermediate = *active;
+
+		DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
+			      pipe_name(pipe),
+			      wm->pipe[pipe].plane[PLANE_PRIMARY],
+			      wm->pipe[pipe].plane[PLANE_CURSOR],
+			      wm->pipe[pipe].plane[PLANE_SPRITE0]);
+	}
+
+	DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
+		      wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
+	DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
+		      wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
+	DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
+		      yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
+}
+
+void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
+{
+	struct intel_plane *plane;
+	struct intel_crtc *crtc;
+
+	mutex_lock(&dev_priv->wm.wm_mutex);
+
+	for_each_intel_plane(&dev_priv->drm, plane) {
+		struct intel_crtc *crtc =
+			intel_get_crtc_for_pipe(dev_priv, plane->pipe);
+		struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+		struct intel_plane_state *plane_state =
+			to_intel_plane_state(plane->base.state);
+		struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
+		enum plane_id plane_id = plane->id;
+		int level;
+
+		if (plane_state->base.visible)
+			continue;
+
+		for (level = 0; level < 3; level++) {
+			struct g4x_pipe_wm *raw =
+				&crtc_state->wm.g4x.raw[level];
+
+			raw->plane[plane_id] = 0;
+			wm_state->wm.plane[plane_id] = 0;
+		}
+
+		if (plane_id == PLANE_PRIMARY) {
+			for (level = 0; level < 3; level++) {
+				struct g4x_pipe_wm *raw =
+					&crtc_state->wm.g4x.raw[level];
+				raw->fbc = 0;
+			}
+
+			wm_state->sr.fbc = 0;
+			wm_state->hpll.fbc = 0;
+			wm_state->fbc_en = false;
+		}
+	}
+
+	for_each_intel_crtc(&dev_priv->drm, crtc) {
+		struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+
+		crtc_state->wm.g4x.intermediate =
+			crtc_state->wm.g4x.optimal;
+		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
+	}
+
+	g4x_program_watermarks(dev_priv);
+
+	mutex_unlock(&dev_priv->wm.wm_mutex);
+}
+
 void vlv_wm_get_hw_state(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -4813,7 +5556,7 @@ void vlv_wm_get_hw_state(struct drm_device *dev)
 		active->cxsr = wm->cxsr;
 
 		for (level = 0; level < active->num_levels; level++) {
-			struct vlv_pipe_wm *raw =
+			struct g4x_pipe_wm *raw =
 				&crtc_state->wm.vlv.raw[level];
 
 			active->sr[level].plane = wm->sr.plane;
@@ -4873,7 +5616,7 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
 			continue;
 
 		for (level = 0; level < wm_state->num_levels; level++) {
-			struct vlv_pipe_wm *raw =
+			struct g4x_pipe_wm *raw =
 				&crtc_state->wm.vlv.raw[level];
 
 			raw->plane[plane_id] = 0;
@@ -7509,7 +8252,7 @@ static void kabylake_init_clock_gating(struct drm_i915_private *dev_priv)
 		I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
 			   GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 
-	/* WaFbcNukeOnHostModify:kbl */
+	/* WaFbcNukeOnHostModify:kbl,cfl */
 	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
 		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
 }
@@ -7977,7 +8720,7 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
 {
 	if (IS_SKYLAKE(dev_priv))
 		dev_priv->display.init_clock_gating = skylake_init_clock_gating;
-	else if (IS_KABYLAKE(dev_priv))
+	else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
 		dev_priv->display.init_clock_gating = kabylake_init_clock_gating;
 	else if (IS_BROXTON(dev_priv))
 		dev_priv->display.init_clock_gating = bxt_init_clock_gating;
@@ -8057,6 +8800,12 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
 		dev_priv->display.initial_watermarks = vlv_initial_watermarks;
 		dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
 		dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
+	} else if (IS_G4X(dev_priv)) {
+		g4x_setup_wm_latency(dev_priv);
+		dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
+		dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
+		dev_priv->display.initial_watermarks = g4x_initial_watermarks;
+		dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
 	} else if (IS_PINEVIEW(dev_priv)) {
 		if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 					    dev_priv->is_ddr3,
@@ -8072,8 +8821,6 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
 			dev_priv->display.update_wm = NULL;
 		} else
 			dev_priv->display.update_wm = pineview_update_wm;
-	} else if (IS_G4X(dev_priv)) {
-		dev_priv->display.update_wm = g4x_update_wm;
 	} else if (IS_GEN4(dev_priv)) {
 		dev_priv->display.update_wm = i965_update_wm;
 	} else if (IS_GEN3(dev_priv)) {
@@ -8156,9 +8903,9 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 	I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
 	I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
 
-	if (intel_wait_for_register_fw(dev_priv,
-				       GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
-				       500)) {
+	if (__intel_wait_for_register_fw(dev_priv,
+					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
+					 500, 0, NULL)) {
 		DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
 		return -ETIMEDOUT;
 	}
@@ -8201,9 +8948,9 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
 	I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
 	I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
 
-	if (intel_wait_for_register_fw(dev_priv,
-				       GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
-				       500)) {
+	if (__intel_wait_for_register_fw(dev_priv,
+					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
+					 500, 0, NULL)) {
 		DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
 		return -ETIMEDOUT;
 	}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 513a0f4b469b..acd1da9b62a3 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -39,17 +39,27 @@
  */
 #define LEGACY_REQUEST_SIZE 200
 
-static int __intel_ring_space(int head, int tail, int size)
+static unsigned int __intel_ring_space(unsigned int head,
+				       unsigned int tail,
+				       unsigned int size)
 {
-	int space = head - tail;
-	if (space <= 0)
-		space += size;
-	return space - I915_RING_FREE_SPACE;
+	/*
+	 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
+	 * same cacheline, the Head Pointer must not be greater than the Tail
+	 * Pointer."
+	 */
+	GEM_BUG_ON(!is_power_of_2(size));
+	return (head - tail - CACHELINE_BYTES) & (size - 1);
 }
 
-void intel_ring_update_space(struct intel_ring *ring)
+unsigned int intel_ring_update_space(struct intel_ring *ring)
 {
-	ring->space = __intel_ring_space(ring->head, ring->emit, ring->size);
+	unsigned int space;
+
+	space = __intel_ring_space(ring->head, ring->emit, ring->size);
+
+	ring->space = space;
+	return space;
 }
 
 static int
@@ -538,9 +548,9 @@ static int init_ring_common(struct intel_engine_cs *engine)
 	I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
 
 	/* If the head is still not zero, the ring is dead */
-	if (intel_wait_for_register_fw(dev_priv, RING_CTL(engine->mmio_base),
-				       RING_VALID, RING_VALID,
-				       50)) {
+	if (intel_wait_for_register(dev_priv, RING_CTL(engine->mmio_base),
+				    RING_VALID, RING_VALID,
+				    50)) {
 		DRM_ERROR("%s initialization failed "
 			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
 			  engine->name,
@@ -1259,6 +1269,8 @@ static int init_phys_status_page(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
+	GEM_BUG_ON(engine->id != RCS);
+
 	dev_priv->status_page_dmah =
 		drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
 	if (!dev_priv->status_page_dmah)
@@ -1270,17 +1282,18 @@ static int init_phys_status_page(struct intel_engine_cs *engine)
 	return 0;
 }
 
-int intel_ring_pin(struct intel_ring *ring, unsigned int offset_bias)
+int intel_ring_pin(struct intel_ring *ring,
+		   struct drm_i915_private *i915,
+		   unsigned int offset_bias)
 {
-	unsigned int flags;
-	enum i915_map_type map;
+	enum i915_map_type map = HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC;
 	struct i915_vma *vma = ring->vma;
+	unsigned int flags;
 	void *addr;
 	int ret;
 
 	GEM_BUG_ON(ring->vaddr);
 
-	map = HAS_LLC(ring->engine->i915) ? I915_MAP_WB : I915_MAP_WC;
 
 	flags = PIN_GLOBAL;
 	if (offset_bias)
@@ -1350,7 +1363,7 @@ intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
 
 	obj = i915_gem_object_create_stolen(dev_priv, size);
 	if (!obj)
-		obj = i915_gem_object_create(dev_priv, size);
+		obj = i915_gem_object_create_internal(dev_priv, size);
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
@@ -1381,8 +1394,6 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 	if (!ring)
 		return ERR_PTR(-ENOMEM);
 
-	ring->engine = engine;
-
 	INIT_LIST_HEAD(&ring->request_list);
 
 	ring->size = size;
@@ -1436,22 +1447,73 @@ static int context_pin(struct i915_gem_context *ctx)
 			    PIN_GLOBAL | PIN_HIGH);
 }
 
-static int intel_ring_context_pin(struct intel_engine_cs *engine,
-				  struct i915_gem_context *ctx)
+static struct i915_vma *
+alloc_context_vma(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *i915 = engine->i915;
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+
+	obj = i915_gem_object_create(i915, engine->context_size);
+	if (IS_ERR(obj))
+		return ERR_CAST(obj);
+
+	/*
+	 * Try to make the context utilize L3 as well as LLC.
+	 *
+	 * On VLV we don't have L3 controls in the PTEs so we
+	 * shouldn't touch the cache level, especially as that
+	 * would make the object snooped which might have a
+	 * negative performance impact.
+	 *
+	 * Snooping is required on non-llc platforms in execlist
+	 * mode, but since all GGTT accesses use PAT entry 0 we
+	 * get snooping anyway regardless of cache_level.
+	 *
+	 * This is only applicable for Ivy Bridge devices since
+	 * later platforms don't have L3 control bits in the PTE.
+	 */
+	if (IS_IVYBRIDGE(i915)) {
+		/* Ignore any error, regard it as a simple optimisation */
+		i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
+	}
+
+	vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
+	if (IS_ERR(vma))
+		i915_gem_object_put(obj);
+
+	return vma;
+}
+
+static struct intel_ring *
+intel_ring_context_pin(struct intel_engine_cs *engine,
+		       struct i915_gem_context *ctx)
 {
 	struct intel_context *ce = &ctx->engine[engine->id];
 	int ret;
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 
-	if (ce->pin_count++)
-		return 0;
+	if (likely(ce->pin_count++))
+		goto out;
 	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
 
+	if (!ce->state && engine->context_size) {
+		struct i915_vma *vma;
+
+		vma = alloc_context_vma(engine);
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
+			goto err;
+		}
+
+		ce->state = vma;
+	}
+
 	if (ce->state) {
 		ret = context_pin(ctx);
 		if (ret)
-			goto error;
+			goto err;
 
 		ce->state->obj->mm.dirty = true;
 	}
@@ -1467,11 +1529,14 @@ static int intel_ring_context_pin(struct intel_engine_cs *engine,
 		ce->initialised = true;
 
 	i915_gem_context_get(ctx);
-	return 0;
 
-error:
+out:
+	/* One ringbuffer to rule them all */
+	return engine->buffer;
+
+err:
 	ce->pin_count = 0;
-	return ret;
+	return ERR_PTR(ret);
 }
 
 static void intel_ring_context_unpin(struct intel_engine_cs *engine,
@@ -1493,78 +1558,70 @@ static void intel_ring_context_unpin(struct intel_engine_cs *engine,
 
 static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
 	struct intel_ring *ring;
-	int ret;
-
-	WARN_ON(engine->buffer);
+	int err;
 
 	intel_engine_setup_common(engine);
 
-	ret = intel_engine_init_common(engine);
-	if (ret)
-		goto error;
+	err = intel_engine_init_common(engine);
+	if (err)
+		goto err;
+
+	if (HWS_NEEDS_PHYSICAL(engine->i915))
+		err = init_phys_status_page(engine);
+	else
+		err = init_status_page(engine);
+	if (err)
+		goto err;
 
 	ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
 	if (IS_ERR(ring)) {
-		ret = PTR_ERR(ring);
-		goto error;
-	}
-
-	if (HWS_NEEDS_PHYSICAL(dev_priv)) {
-		WARN_ON(engine->id != RCS);
-		ret = init_phys_status_page(engine);
-		if (ret)
-			goto error;
-	} else {
-		ret = init_status_page(engine);
-		if (ret)
-			goto error;
+		err = PTR_ERR(ring);
+		goto err_hws;
 	}
 
 	/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
-	ret = intel_ring_pin(ring, I915_GTT_PAGE_SIZE);
-	if (ret) {
-		intel_ring_free(ring);
-		goto error;
-	}
+	err = intel_ring_pin(ring, engine->i915, I915_GTT_PAGE_SIZE);
+	if (err)
+		goto err_ring;
+
+	GEM_BUG_ON(engine->buffer);
 	engine->buffer = ring;
 
 	return 0;
 
-error:
-	intel_engine_cleanup(engine);
-	return ret;
+err_ring:
+	intel_ring_free(ring);
+err_hws:
+	if (HWS_NEEDS_PHYSICAL(engine->i915))
+		cleanup_phys_status_page(engine);
+	else
+		cleanup_status_page(engine);
+err:
+	intel_engine_cleanup_common(engine);
+	return err;
 }
 
 void intel_engine_cleanup(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv;
-
-	dev_priv = engine->i915;
+	struct drm_i915_private *dev_priv = engine->i915;
 
-	if (engine->buffer) {
-		WARN_ON(INTEL_GEN(dev_priv) > 2 &&
-			(I915_READ_MODE(engine) & MODE_IDLE) == 0);
+	WARN_ON(INTEL_GEN(dev_priv) > 2 &&
+		(I915_READ_MODE(engine) & MODE_IDLE) == 0);
 
-		intel_ring_unpin(engine->buffer);
-		intel_ring_free(engine->buffer);
-		engine->buffer = NULL;
-	}
+	intel_ring_unpin(engine->buffer);
+	intel_ring_free(engine->buffer);
 
 	if (engine->cleanup)
 		engine->cleanup(engine);
 
-	if (HWS_NEEDS_PHYSICAL(dev_priv)) {
-		WARN_ON(engine->id != RCS);
+	if (HWS_NEEDS_PHYSICAL(dev_priv))
 		cleanup_phys_status_page(engine);
-	} else {
+	else
 		cleanup_status_page(engine);
-	}
 
 	intel_engine_cleanup_common(engine);
 
-	engine->i915 = NULL;
 	dev_priv->engine[engine->id] = NULL;
 	kfree(engine);
 }
@@ -1591,9 +1648,6 @@ static int ring_request_alloc(struct drm_i915_gem_request *request)
 	 */
 	request->reserved_space += LEGACY_REQUEST_SIZE;
 
-	GEM_BUG_ON(!request->engine->buffer);
-	request->ring = request->engine->buffer;
-
 	cs = intel_ring_begin(request, 0);
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
@@ -1602,7 +1656,8 @@ static int ring_request_alloc(struct drm_i915_gem_request *request)
 	return 0;
 }
 
-static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
+static noinline int wait_for_space(struct drm_i915_gem_request *req,
+				   unsigned int bytes)
 {
 	struct intel_ring *ring = req->ring;
 	struct drm_i915_gem_request *target;
@@ -1610,8 +1665,7 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 
 	lockdep_assert_held(&req->i915->drm.struct_mutex);
 
-	intel_ring_update_space(ring);
-	if (ring->space >= bytes)
+	if (intel_ring_update_space(ring) >= bytes)
 		return 0;
 
 	/*
@@ -1626,12 +1680,9 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 	GEM_BUG_ON(!req->reserved_space);
 
 	list_for_each_entry(target, &ring->request_list, ring_link) {
-		unsigned space;
-
 		/* Would completion of this request free enough space? */
-		space = __intel_ring_space(target->postfix, ring->emit,
-					   ring->size);
-		if (space >= bytes)
+		if (bytes <= __intel_ring_space(target->postfix,
+						ring->emit, ring->size))
 			break;
 	}
 
@@ -1651,59 +1702,64 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 	return 0;
 }
 
-u32 *intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
+u32 *intel_ring_begin(struct drm_i915_gem_request *req,
+		      unsigned int num_dwords)
 {
 	struct intel_ring *ring = req->ring;
-	int remain_actual = ring->size - ring->emit;
-	int remain_usable = ring->effective_size - ring->emit;
-	int bytes = num_dwords * sizeof(u32);
-	int total_bytes, wait_bytes;
-	bool need_wrap = false;
+	const unsigned int remain_usable = ring->effective_size - ring->emit;
+	const unsigned int bytes = num_dwords * sizeof(u32);
+	unsigned int need_wrap = 0;
+	unsigned int total_bytes;
 	u32 *cs;
 
 	total_bytes = bytes + req->reserved_space;
+	GEM_BUG_ON(total_bytes > ring->effective_size);
 
-	if (unlikely(bytes > remain_usable)) {
-		/*
-		 * Not enough space for the basic request. So need to flush
-		 * out the remainder and then wait for base + reserved.
-		 */
-		wait_bytes = remain_actual + total_bytes;
-		need_wrap = true;
-	} else if (unlikely(total_bytes > remain_usable)) {
-		/*
-		 * The base request will fit but the reserved space
-		 * falls off the end. So we don't need an immediate wrap
-		 * and only need to effectively wait for the reserved
-		 * size space from the start of ringbuffer.
-		 */
-		wait_bytes = remain_actual + req->reserved_space;
-	} else {
-		/* No wrapping required, just waiting. */
-		wait_bytes = total_bytes;
+	if (unlikely(total_bytes > remain_usable)) {
+		const int remain_actual = ring->size - ring->emit;
+
+		if (bytes > remain_usable) {
+			/*
+			 * Not enough space for the basic request. So need to
+			 * flush out the remainder and then wait for
+			 * base + reserved.
+			 */
+			total_bytes += remain_actual;
+			need_wrap = remain_actual | 1;
+		} else  {
+			/*
+			 * The base request will fit but the reserved space
+			 * falls off the end. So we don't need an immediate
+			 * wrap and only need to effectively wait for the
+			 * reserved size from the start of ringbuffer.
+			 */
+			total_bytes = req->reserved_space + remain_actual;
+		}
 	}
 
-	if (wait_bytes > ring->space) {
-		int ret = wait_for_space(req, wait_bytes);
+	if (unlikely(total_bytes > ring->space)) {
+		int ret = wait_for_space(req, total_bytes);
 		if (unlikely(ret))
 			return ERR_PTR(ret);
 	}
 
 	if (unlikely(need_wrap)) {
-		GEM_BUG_ON(remain_actual > ring->space);
-		GEM_BUG_ON(ring->emit + remain_actual > ring->size);
+		need_wrap &= ~1;
+		GEM_BUG_ON(need_wrap > ring->space);
+		GEM_BUG_ON(ring->emit + need_wrap > ring->size);
 
 		/* Fill the tail with MI_NOOP */
-		memset(ring->vaddr + ring->emit, 0, remain_actual);
+		memset(ring->vaddr + ring->emit, 0, need_wrap);
 		ring->emit = 0;
-		ring->space -= remain_actual;
+		ring->space -= need_wrap;
 	}
 
 	GEM_BUG_ON(ring->emit > ring->size - bytes);
+	GEM_BUG_ON(ring->space < bytes);
 	cs = ring->vaddr + ring->emit;
+	GEM_DEBUG_EXEC(memset(cs, POISON_INUSE, bytes));
 	ring->emit += bytes;
 	ring->space -= bytes;
-	GEM_BUG_ON(ring->space < 0);
 
 	return cs;
 }
@@ -1749,11 +1805,11 @@ static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
 	I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
 
 	/* Wait for the ring not to be idle, i.e. for it to wake up. */
-	if (intel_wait_for_register_fw(dev_priv,
-				       GEN6_BSD_SLEEP_PSMI_CONTROL,
-				       GEN6_BSD_SLEEP_INDICATOR,
-				       0,
-				       50))
+	if (__intel_wait_for_register_fw(dev_priv,
+					 GEN6_BSD_SLEEP_PSMI_CONTROL,
+					 GEN6_BSD_SLEEP_INDICATOR,
+					 0,
+					 1000, 0, NULL))
 		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
 
 	/* Now that the ring is fully powered up, update the tail */
@@ -2195,20 +2251,6 @@ int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
 	return intel_init_ring_buffer(engine);
 }
 
-/**
- * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
- */
-int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	intel_ring_default_vfuncs(dev_priv, engine);
-
-	engine->emit_flush = gen6_bsd_ring_flush;
-
-	return intel_init_ring_buffer(engine);
-}
-
 int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index f7144fe09613..6aa20ac8cde3 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -17,17 +17,6 @@
 #define CACHELINE_BYTES 64
 #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
 
-/*
- * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
- * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
- * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
- *
- * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
- * cacheline, the Head Pointer must not be greater than the Tail
- * Pointer."
- */
-#define I915_RING_FREE_SPACE 64
-
 struct intel_hw_status_page {
 	struct i915_vma *vma;
 	u32 *page_addr;
@@ -139,17 +128,15 @@ struct intel_ring {
 	struct i915_vma *vma;
 	void *vaddr;
 
-	struct intel_engine_cs *engine;
-
 	struct list_head request_list;
 
 	u32 head;
 	u32 tail;
 	u32 emit;
 
-	int space;
-	int size;
-	int effective_size;
+	u32 space;
+	u32 size;
+	u32 effective_size;
 };
 
 struct i915_gem_context;
@@ -190,15 +177,28 @@ enum intel_engine_id {
 	VECS
 };
 
+struct i915_priolist {
+	struct rb_node node;
+	struct list_head requests;
+	int priority;
+};
+
+#define INTEL_ENGINE_CS_MAX_NAME 8
+
 struct intel_engine_cs {
 	struct drm_i915_private *i915;
-	const char	*name;
+	char name[INTEL_ENGINE_CS_MAX_NAME];
 	enum intel_engine_id id;
-	unsigned int exec_id;
+	unsigned int uabi_id;
 	unsigned int hw_id;
 	unsigned int guc_id;
-	u32		mmio_base;
+
+	u8 class;
+	u8 instance;
+	u32 context_size;
+	u32 mmio_base;
 	unsigned int irq_shift;
+
 	struct intel_ring *buffer;
 	struct intel_timeline *timeline;
 
@@ -266,8 +266,8 @@ struct intel_engine_cs {
 
 	void		(*set_default_submission)(struct intel_engine_cs *engine);
 
-	int		(*context_pin)(struct intel_engine_cs *engine,
-				       struct i915_gem_context *ctx);
+	struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
+					  struct i915_gem_context *ctx);
 	void		(*context_unpin)(struct intel_engine_cs *engine,
 					 struct i915_gem_context *ctx);
 	int		(*request_alloc)(struct drm_i915_gem_request *req);
@@ -373,9 +373,18 @@ struct intel_engine_cs {
 
 	/* Execlists */
 	struct tasklet_struct irq_tasklet;
+	struct i915_priolist default_priolist;
+	bool no_priolist;
 	struct execlist_port {
-		struct drm_i915_gem_request *request;
-		unsigned int count;
+		struct drm_i915_gem_request *request_count;
+#define EXECLIST_COUNT_BITS 2
+#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
+#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
+#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
+#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
+#define port_set(p, packed) ((p)->request_count = (packed))
+#define port_isset(p) ((p)->request_count)
+#define port_index(p, e) ((p) - (e)->execlist_port)
 		GEM_DEBUG_DECL(u32 context_id);
 	} execlist_port[2];
 	struct rb_root execlist_queue;
@@ -488,9 +497,11 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
 
 struct intel_ring *
 intel_engine_create_ring(struct intel_engine_cs *engine, int size);
-int intel_ring_pin(struct intel_ring *ring, unsigned int offset_bias);
+int intel_ring_pin(struct intel_ring *ring,
+		   struct drm_i915_private *i915,
+		   unsigned int offset_bias);
 void intel_ring_reset(struct intel_ring *ring, u32 tail);
-void intel_ring_update_space(struct intel_ring *ring);
+unsigned int intel_ring_update_space(struct intel_ring *ring);
 void intel_ring_unpin(struct intel_ring *ring);
 void intel_ring_free(struct intel_ring *ring);
 
@@ -501,7 +512,8 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
 
 int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
 
-u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req, int n);
+u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
+				   unsigned int n);
 
 static inline void
 intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
@@ -541,6 +553,25 @@ assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
 	 */
 	GEM_BUG_ON(!IS_ALIGNED(tail, 8));
 	GEM_BUG_ON(tail >= ring->size);
+
+	/*
+	 * "Ring Buffer Use"
+	 *	Gen2 BSpec "1. Programming Environment" / 1.4.4.6
+	 *	Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5
+	 *	Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5
+	 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
+	 * same cacheline, the Head Pointer must not be greater than the Tail
+	 * Pointer."
+	 *
+	 * We use ring->head as the last known location of the actual RING_HEAD,
+	 * it may have advanced but in the worst case it is equally the same
+	 * as ring->head and so we should never program RING_TAIL to advance
+	 * into the same cacheline as ring->head.
+	 */
+#define cacheline(a) round_down(a, CACHELINE_BYTES)
+	GEM_BUG_ON(cacheline(tail) == cacheline(ring->head) &&
+		   tail < ring->head);
+#undef cacheline
 }
 
 static inline unsigned int
@@ -566,7 +597,6 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine);
 
 int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
-int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
 
@@ -667,7 +697,8 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine,
 			   struct intel_wait *wait);
 void intel_engine_remove_wait(struct intel_engine_cs *engine,
 			      struct intel_wait *wait);
-void intel_engine_enable_signaling(struct drm_i915_gem_request *request);
+void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
+				   bool wakeup);
 void intel_engine_cancel_signaling(struct drm_i915_gem_request *request);
 
 static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
@@ -700,6 +731,7 @@ static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
 bool intel_engine_is_idle(struct intel_engine_cs *engine);
 bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
 
+void intel_engines_mark_idle(struct drm_i915_private *i915);
 void intel_engines_reset_default_submission(struct drm_i915_private *i915);
 
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index f8a375f8dde6..efe80ed5fd4d 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -494,6 +494,55 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
 	BIT_ULL(POWER_DOMAIN_INIT))
 
+#define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_A_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_B_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_C_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_D_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_MODESET) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 {
 	WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
@@ -762,13 +811,14 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 		}
 		break;
 	case SKL_DISP_PW_MISC_IO:
-	case SKL_DISP_PW_DDI_A_E: /* GLK_DISP_PW_DDI_A */
+	case SKL_DISP_PW_DDI_A_E: /* GLK_DISP_PW_DDI_A, CNL_DISP_PW_DDI_A */
 	case SKL_DISP_PW_DDI_B:
 	case SKL_DISP_PW_DDI_C:
 	case SKL_DISP_PW_DDI_D:
-	case GLK_DISP_PW_AUX_A:
-	case GLK_DISP_PW_AUX_B:
-	case GLK_DISP_PW_AUX_C:
+	case GLK_DISP_PW_AUX_A: /* CNL_DISP_PW_AUX_A */
+	case GLK_DISP_PW_AUX_B: /* CNL_DISP_PW_AUX_B */
+	case GLK_DISP_PW_AUX_C: /* CNL_DISP_PW_AUX_C */
+	case CNL_DISP_PW_AUX_D:
 		break;
 	default:
 		WARN(1, "Unknown power well %lu\n", power_well->id);
@@ -803,8 +853,7 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 			DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
 		}
 
-		if (IS_GEN9(dev_priv))
-			gen9_sanitize_power_well_requests(dev_priv, power_well);
+		gen9_sanitize_power_well_requests(dev_priv, power_well);
 	}
 
 	if (wait_for(!!(I915_READ(HSW_PWR_WELL_DRIVER) & state_mask) == enable,
@@ -992,6 +1041,38 @@ static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 	return true;
 }
 
+static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
+					 struct i915_power_well *power_well)
+{
+	if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
+		i830_enable_pipe(dev_priv, PIPE_A);
+	if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
+		i830_enable_pipe(dev_priv, PIPE_B);
+}
+
+static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
+					  struct i915_power_well *power_well)
+{
+	i830_disable_pipe(dev_priv, PIPE_B);
+	i830_disable_pipe(dev_priv, PIPE_A);
+}
+
+static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
+					  struct i915_power_well *power_well)
+{
+	return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
+		I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
+}
+
+static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
+					  struct i915_power_well *power_well)
+{
+	if (power_well->count > 0)
+		i830_pipes_power_well_enable(dev_priv, power_well);
+	else
+		i830_pipes_power_well_disable(dev_priv, power_well);
+}
+
 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 			       struct i915_power_well *power_well, bool enable)
 {
@@ -1880,6 +1961,15 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_AUX_D) |		\
 	BIT_ULL(POWER_DOMAIN_INIT))
 
+#define I830_PIPES_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PIPE_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |	\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |	\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |	\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |	\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
 	.sync_hw = i9xx_power_well_sync_hw_noop,
 	.enable = i9xx_always_on_power_well_noop,
@@ -1910,6 +2000,27 @@ static struct i915_power_well i9xx_always_on_power_well[] = {
 	},
 };
 
+static const struct i915_power_well_ops i830_pipes_power_well_ops = {
+	.sync_hw = i830_pipes_power_well_sync_hw,
+	.enable = i830_pipes_power_well_enable,
+	.disable = i830_pipes_power_well_disable,
+	.is_enabled = i830_pipes_power_well_enabled,
+};
+
+static struct i915_power_well i830_power_wells[] = {
+	{
+		.name = "always-on",
+		.always_on = 1,
+		.domains = POWER_DOMAIN_MASK,
+		.ops = &i9xx_always_on_power_well_ops,
+	},
+	{
+		.name = "pipes",
+		.domains = I830_PIPES_POWER_DOMAINS,
+		.ops = &i830_pipes_power_well_ops,
+	},
+};
+
 static const struct i915_power_well_ops hsw_power_well_ops = {
 	.sync_hw = hsw_power_well_sync_hw,
 	.enable = hsw_power_well_enable,
@@ -2275,6 +2386,82 @@ static struct i915_power_well glk_power_wells[] = {
 	},
 };
 
+static struct i915_power_well cnl_power_wells[] = {
+	{
+		.name = "always-on",
+		.always_on = 1,
+		.domains = POWER_DOMAIN_MASK,
+		.ops = &i9xx_always_on_power_well_ops,
+	},
+	{
+		.name = "power well 1",
+		/* Handled by the DMC firmware */
+		.domains = 0,
+		.ops = &skl_power_well_ops,
+		.id = SKL_DISP_PW_1,
+	},
+	{
+		.name = "AUX A",
+		.domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = CNL_DISP_PW_AUX_A,
+	},
+	{
+		.name = "AUX B",
+		.domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = CNL_DISP_PW_AUX_B,
+	},
+	{
+		.name = "AUX C",
+		.domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = CNL_DISP_PW_AUX_C,
+	},
+	{
+		.name = "AUX D",
+		.domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = CNL_DISP_PW_AUX_D,
+	},
+	{
+		.name = "DC off",
+		.domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
+		.ops = &gen9_dc_off_power_well_ops,
+		.id = SKL_DISP_PW_DC_OFF,
+	},
+	{
+		.name = "power well 2",
+		.domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = SKL_DISP_PW_2,
+	},
+	{
+		.name = "DDI A IO power well",
+		.domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = CNL_DISP_PW_DDI_A,
+	},
+	{
+		.name = "DDI B IO power well",
+		.domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = SKL_DISP_PW_DDI_B,
+	},
+	{
+		.name = "DDI C IO power well",
+		.domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = SKL_DISP_PW_DDI_C,
+	},
+	{
+		.name = "DDI D IO power well",
+		.domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
+		.ops = &skl_power_well_ops,
+		.id = SKL_DISP_PW_DDI_D,
+	},
+};
+
 static int
 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
 				   int disable_power_well)
@@ -2369,6 +2556,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 		set_power_wells(power_domains, bdw_power_wells);
 	} else if (IS_GEN9_BC(dev_priv)) {
 		set_power_wells(power_domains, skl_power_wells);
+	} else if (IS_CANNONLAKE(dev_priv)) {
+		set_power_wells(power_domains, cnl_power_wells);
 	} else if (IS_BROXTON(dev_priv)) {
 		set_power_wells(power_domains, bxt_power_wells);
 	} else if (IS_GEMINILAKE(dev_priv)) {
@@ -2377,6 +2566,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 		set_power_wells(power_domains, chv_power_wells);
 	} else if (IS_VALLEYVIEW(dev_priv)) {
 		set_power_wells(power_domains, vlv_power_wells);
+	} else if (IS_I830(dev_priv)) {
+		set_power_wells(power_domains, i830_power_wells);
 	} else {
 		set_power_wells(power_domains, i9xx_always_on_power_well);
 	}
@@ -2569,6 +2760,111 @@ void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
 	mutex_unlock(&power_domains->lock);
 }
 
+#define CNL_PROCMON_IDX(val) \
+	(((val) & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) >> VOLTAGE_INFO_SHIFT)
+#define NUM_CNL_PROCMON \
+	(CNL_PROCMON_IDX(VOLTAGE_INFO_MASK | PROCESS_INFO_MASK) + 1)
+
+static const struct cnl_procmon {
+	u32 dw1, dw9, dw10;
+} cnl_procmon_values[NUM_CNL_PROCMON] = {
+	[CNL_PROCMON_IDX(VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0)] =
+		{ .dw1 = 0x00 << 16, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
+	[CNL_PROCMON_IDX(VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0)] =
+		{ .dw1 = 0x00 << 16, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
+	[CNL_PROCMON_IDX(VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1)] =
+		{ .dw1 = 0x00 << 16, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
+	[CNL_PROCMON_IDX(VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0)] =
+		{ .dw1 = 0x00 << 16, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
+	[CNL_PROCMON_IDX(VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1)] =
+		{ .dw1 = 0x44 << 16, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
+};
+
+static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
+{
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	const struct cnl_procmon *procmon;
+	struct i915_power_well *well;
+	u32 val;
+
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+	/* 1. Enable PCH Reset Handshake */
+	val = I915_READ(HSW_NDE_RSTWRN_OPT);
+	val |= RESET_PCH_HANDSHAKE_ENABLE;
+	I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
+
+	/* 2. Enable Comp */
+	val = I915_READ(CHICKEN_MISC_2);
+	val &= ~COMP_PWR_DOWN;
+	I915_WRITE(CHICKEN_MISC_2, val);
+
+	val = I915_READ(CNL_PORT_COMP_DW3);
+	procmon = &cnl_procmon_values[CNL_PROCMON_IDX(val)];
+
+	WARN_ON(procmon->dw10 == 0);
+
+	val = I915_READ(CNL_PORT_COMP_DW1);
+	val &= ~((0xff << 16) | 0xff);
+	val |= procmon->dw1;
+	I915_WRITE(CNL_PORT_COMP_DW1, val);
+
+	I915_WRITE(CNL_PORT_COMP_DW9, procmon->dw9);
+	I915_WRITE(CNL_PORT_COMP_DW10, procmon->dw10);
+
+	val = I915_READ(CNL_PORT_COMP_DW0);
+	val |= COMP_INIT;
+	I915_WRITE(CNL_PORT_COMP_DW0, val);
+
+	/* 3. */
+	val = I915_READ(CNL_PORT_CL1CM_DW5);
+	val |= CL_POWER_DOWN_ENABLE;
+	I915_WRITE(CNL_PORT_CL1CM_DW5, val);
+
+	/* 4. Enable Power Well 1 (PG1) and Aux IO Power */
+	mutex_lock(&power_domains->lock);
+	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
+	intel_power_well_enable(dev_priv, well);
+	mutex_unlock(&power_domains->lock);
+
+	/* 5. Enable CD clock */
+	cnl_init_cdclk(dev_priv);
+
+	/* 6. Enable DBUF */
+	gen9_dbuf_enable(dev_priv);
+}
+
+#undef CNL_PROCMON_IDX
+#undef NUM_CNL_PROCMON
+
+static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
+{
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_well *well;
+	u32 val;
+
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+	/* 1. Disable all display engine functions -> aready done */
+
+	/* 2. Disable DBUF */
+	gen9_dbuf_disable(dev_priv);
+
+	/* 3. Disable CD clock */
+	cnl_uninit_cdclk(dev_priv);
+
+	/* 4. Disable Power Well 1 (PG1) and Aux IO Power */
+	mutex_lock(&power_domains->lock);
+	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
+	intel_power_well_disable(dev_priv, well);
+	mutex_unlock(&power_domains->lock);
+
+	/* 5. Disable Comp */
+	val = I915_READ(CHICKEN_MISC_2);
+	val |= COMP_PWR_DOWN;
+	I915_WRITE(CHICKEN_MISC_2, val);
+}
+
 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
 {
 	struct i915_power_well *cmn_bc =
@@ -2701,7 +2997,9 @@ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
 
 	power_domains->initializing = true;
 
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_CANNONLAKE(dev_priv)) {
+		cnl_display_core_init(dev_priv, resume);
+	} else if (IS_GEN9_BC(dev_priv)) {
 		skl_display_core_init(dev_priv, resume);
 	} else if (IS_GEN9_LP(dev_priv)) {
 		bxt_display_core_init(dev_priv, resume);
@@ -2740,7 +3038,9 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
 	if (!i915.disable_power_well)
 		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 
-	if (IS_GEN9_BC(dev_priv))
+	if (IS_CANNONLAKE(dev_priv))
+		cnl_display_core_uninit(dev_priv);
+	else if (IS_GEN9_BC(dev_priv))
 		skl_display_core_uninit(dev_priv);
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_display_core_uninit(dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 816a6f5a3fd9..3f8f30b412cd 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -100,18 +100,6 @@ struct intel_sdvo {
 	uint16_t hotplug_active;
 
 	/**
-	 * This is used to select the color range of RBG outputs in HDMI mode.
-	 * It is only valid when using TMDS encoding and 8 bit per color mode.
-	 */
-	uint32_t color_range;
-	bool color_range_auto;
-
-	/**
-	 * HDMI user specified aspect ratio
-	 */
-	enum hdmi_picture_aspect aspect_ratio;
-
-	/**
 	 * This is set if we're going to treat the device as TV-out.
 	 *
 	 * While we have these nice friendly flags for output types that ought
@@ -122,9 +110,6 @@ struct intel_sdvo {
 
 	enum port port;
 
-	/* This is for current tv format name */
-	int tv_format_index;
-
 	/**
 	 * This is set if we treat the device as HDMI, instead of DVI.
 	 */
@@ -159,8 +144,6 @@ struct intel_sdvo_connector {
 	/* Mark the type of connector */
 	uint16_t output_flag;
 
-	enum hdmi_force_audio force_audio;
-
 	/* This contains all current supported TV format */
 	u8 tv_format_supported[TV_FORMAT_NUM];
 	int   format_supported_num;
@@ -187,24 +170,19 @@ struct intel_sdvo_connector {
 	/* add the property for the SDVO-TV/LVDS */
 	struct drm_property *brightness;
 
-	/* Add variable to record current setting for the above property */
-	u32	left_margin, right_margin, top_margin, bottom_margin;
-
 	/* this is to get the range of margin.*/
-	u32	max_hscan,  max_vscan;
-	u32	max_hpos, cur_hpos;
-	u32	max_vpos, cur_vpos;
-	u32	cur_brightness, max_brightness;
-	u32	cur_contrast,	max_contrast;
-	u32	cur_saturation, max_saturation;
-	u32	cur_hue,	max_hue;
-	u32	cur_sharpness,	max_sharpness;
-	u32	cur_flicker_filter,		max_flicker_filter;
-	u32	cur_flicker_filter_adaptive,	max_flicker_filter_adaptive;
-	u32	cur_flicker_filter_2d,		max_flicker_filter_2d;
-	u32	cur_tv_chroma_filter,	max_tv_chroma_filter;
-	u32	cur_tv_luma_filter,	max_tv_luma_filter;
-	u32	cur_dot_crawl,	max_dot_crawl;
+	u32 max_hscan, max_vscan;
+};
+
+struct intel_sdvo_connector_state {
+	/* base.base: tv.saturation/contrast/hue/brightness */
+	struct intel_digital_connector_state base;
+
+	struct {
+		unsigned overscan_h, overscan_v, hpos, vpos, sharpness;
+		unsigned flicker_filter, flicker_filter_2d, flicker_filter_adaptive;
+		unsigned chroma_filter, luma_filter, dot_crawl;
+	} tv;
 };
 
 static struct intel_sdvo *to_sdvo(struct intel_encoder *encoder)
@@ -217,9 +195,16 @@ static struct intel_sdvo *intel_attached_sdvo(struct drm_connector *connector)
 	return to_sdvo(intel_attached_encoder(connector));
 }
 
-static struct intel_sdvo_connector *to_intel_sdvo_connector(struct drm_connector *connector)
+static struct intel_sdvo_connector *
+to_intel_sdvo_connector(struct drm_connector *connector)
+{
+	return container_of(connector, struct intel_sdvo_connector, base.base);
+}
+
+static struct intel_sdvo_connector_state *
+to_intel_sdvo_connector_state(struct drm_connector_state *conn_state)
 {
-	return container_of(to_intel_connector(connector), struct intel_sdvo_connector, base);
+	return container_of(conn_state, struct intel_sdvo_connector_state, base.base);
 }
 
 static bool
@@ -1035,12 +1020,13 @@ static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
 					  sdvo_data, sizeof(sdvo_data));
 }
 
-static bool intel_sdvo_set_tv_format(struct intel_sdvo *intel_sdvo)
+static bool intel_sdvo_set_tv_format(struct intel_sdvo *intel_sdvo,
+				     struct drm_connector_state *conn_state)
 {
 	struct intel_sdvo_tv_format format;
 	uint32_t format_map;
 
-	format_map = 1 << intel_sdvo->tv_format_index;
+	format_map = 1 << conn_state->tv.mode;
 	memset(&format, 0, sizeof(format));
 	memcpy(&format, &format_map, min(sizeof(format), sizeof(format_map)));
 
@@ -1127,6 +1113,8 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 				      struct drm_connector_state *conn_state)
 {
 	struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
+	struct intel_sdvo_connector_state *intel_sdvo_state =
+		to_intel_sdvo_connector_state(conn_state);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	struct drm_display_mode *mode = &pipe_config->base.mode;
 
@@ -1165,9 +1153,14 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 	pipe_config->pixel_multiplier =
 		intel_sdvo_get_pixel_multiplier(adjusted_mode);
 
-	pipe_config->has_hdmi_sink = intel_sdvo->has_hdmi_monitor;
+	if (intel_sdvo_state->base.force_audio != HDMI_AUDIO_OFF_DVI)
+		pipe_config->has_hdmi_sink = intel_sdvo->has_hdmi_monitor;
 
-	if (intel_sdvo->color_range_auto) {
+	if (intel_sdvo_state->base.force_audio == HDMI_AUDIO_ON ||
+	    (intel_sdvo_state->base.force_audio == HDMI_AUDIO_AUTO && intel_sdvo->has_hdmi_audio))
+		pipe_config->has_audio = true;
+
+	if (intel_sdvo_state->base.broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) {
 		/* See CEA-861-E - 5.1 Default Encoding Parameters */
 		/* FIXME: This bit is only valid when using TMDS encoding and 8
 		 * bit per color mode. */
@@ -1176,7 +1169,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 			pipe_config->limited_color_range = true;
 	} else {
 		if (pipe_config->has_hdmi_sink &&
-		    intel_sdvo->color_range == HDMI_COLOR_RANGE_16_235)
+		    intel_sdvo_state->base.broadcast_rgb == INTEL_BROADCAST_RGB_LIMITED)
 			pipe_config->limited_color_range = true;
 	}
 
@@ -1186,11 +1179,73 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 
 	/* Set user selected PAR to incoming mode's member */
 	if (intel_sdvo->is_hdmi)
-		adjusted_mode->picture_aspect_ratio = intel_sdvo->aspect_ratio;
+		adjusted_mode->picture_aspect_ratio = conn_state->picture_aspect_ratio;
 
 	return true;
 }
 
+#define UPDATE_PROPERTY(input, NAME) \
+	do { \
+		val = input; \
+		intel_sdvo_set_value(intel_sdvo, SDVO_CMD_SET_##NAME, &val, sizeof(val)); \
+	} while (0)
+
+static void intel_sdvo_update_props(struct intel_sdvo *intel_sdvo,
+				    struct intel_sdvo_connector_state *sdvo_state)
+{
+	struct drm_connector_state *conn_state = &sdvo_state->base.base;
+	struct intel_sdvo_connector *intel_sdvo_conn =
+		to_intel_sdvo_connector(conn_state->connector);
+	uint16_t val;
+
+	if (intel_sdvo_conn->left)
+		UPDATE_PROPERTY(sdvo_state->tv.overscan_h, OVERSCAN_H);
+
+	if (intel_sdvo_conn->top)
+		UPDATE_PROPERTY(sdvo_state->tv.overscan_v, OVERSCAN_V);
+
+	if (intel_sdvo_conn->hpos)
+		UPDATE_PROPERTY(sdvo_state->tv.hpos, HPOS);
+
+	if (intel_sdvo_conn->vpos)
+		UPDATE_PROPERTY(sdvo_state->tv.vpos, VPOS);
+
+	if (intel_sdvo_conn->saturation)
+		UPDATE_PROPERTY(conn_state->tv.saturation, SATURATION);
+
+	if (intel_sdvo_conn->contrast)
+		UPDATE_PROPERTY(conn_state->tv.contrast, CONTRAST);
+
+	if (intel_sdvo_conn->hue)
+		UPDATE_PROPERTY(conn_state->tv.hue, HUE);
+
+	if (intel_sdvo_conn->brightness)
+		UPDATE_PROPERTY(conn_state->tv.brightness, BRIGHTNESS);
+
+	if (intel_sdvo_conn->sharpness)
+		UPDATE_PROPERTY(sdvo_state->tv.sharpness, SHARPNESS);
+
+	if (intel_sdvo_conn->flicker_filter)
+		UPDATE_PROPERTY(sdvo_state->tv.flicker_filter, FLICKER_FILTER);
+
+	if (intel_sdvo_conn->flicker_filter_2d)
+		UPDATE_PROPERTY(sdvo_state->tv.flicker_filter_2d, FLICKER_FILTER_2D);
+
+	if (intel_sdvo_conn->flicker_filter_adaptive)
+		UPDATE_PROPERTY(sdvo_state->tv.flicker_filter_adaptive, FLICKER_FILTER_ADAPTIVE);
+
+	if (intel_sdvo_conn->tv_chroma_filter)
+		UPDATE_PROPERTY(sdvo_state->tv.chroma_filter, TV_CHROMA_FILTER);
+
+	if (intel_sdvo_conn->tv_luma_filter)
+		UPDATE_PROPERTY(sdvo_state->tv.luma_filter, TV_LUMA_FILTER);
+
+	if (intel_sdvo_conn->dot_crawl)
+		UPDATE_PROPERTY(sdvo_state->tv.dot_crawl, DOT_CRAWL);
+
+#undef UPDATE_PROPERTY
+}
+
 static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 				  struct intel_crtc_state *crtc_state,
 				  struct drm_connector_state *conn_state)
@@ -1198,6 +1253,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
+	struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state(conn_state);
 	struct drm_display_mode *mode = &crtc_state->base.mode;
 	struct intel_sdvo *intel_sdvo = to_sdvo(intel_encoder);
 	u32 sdvox;
@@ -1205,6 +1261,8 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 	struct intel_sdvo_dtd input_dtd, output_dtd;
 	int rate;
 
+	intel_sdvo_update_props(intel_sdvo, sdvo_state);
+
 	/* First, set the input mapping for the first input to our controlled
 	 * output. This is only correct if we're a single-input device, in
 	 * which case the first input is the output from the appropriate SDVO
@@ -1246,7 +1304,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 		intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI);
 
 	if (intel_sdvo->is_tv &&
-	    !intel_sdvo_set_tv_format(intel_sdvo))
+	    !intel_sdvo_set_tv_format(intel_sdvo, conn_state))
 		return;
 
 	intel_sdvo_get_dtd_from_mode(&input_dtd, adjusted_mode);
@@ -1290,7 +1348,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 	else
 		sdvox |= SDVO_PIPE_SEL(crtc->pipe);
 
-	if (intel_sdvo->has_hdmi_audio)
+	if (crtc_state->has_audio)
 		sdvox |= SDVO_AUDIO_ENABLE;
 
 	if (INTEL_GEN(dev_priv) >= 4) {
@@ -1699,12 +1757,6 @@ intel_sdvo_tmds_sink_detect(struct drm_connector *connector)
 		kfree(edid);
 	}
 
-	if (status == connector_status_connected) {
-		struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
-		if (intel_sdvo_connector->force_audio != HDMI_AUDIO_AUTO)
-			intel_sdvo->has_hdmi_audio = (intel_sdvo_connector->force_audio == HDMI_AUDIO_ON);
-	}
-
 	return status;
 }
 
@@ -1884,6 +1936,7 @@ static const struct drm_display_mode sdvo_tv_modes[] = {
 static void intel_sdvo_get_tv_modes(struct drm_connector *connector)
 {
 	struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
+	const struct drm_connector_state *conn_state = connector->state;
 	struct intel_sdvo_sdtv_resolution_request tv_res;
 	uint32_t reply = 0, format_map = 0;
 	int i;
@@ -1894,7 +1947,7 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector)
 	/* Read the list of supported input resolutions for the selected TV
 	 * format.
 	 */
-	format_map = 1 << intel_sdvo->tv_format_index;
+	format_map = 1 << conn_state->tv.mode;
 	memcpy(&tv_res, &format_map,
 	       min(sizeof(format_map), sizeof(struct intel_sdvo_sdtv_resolution_request)));
 
@@ -1983,204 +2036,121 @@ static void intel_sdvo_destroy(struct drm_connector *connector)
 	kfree(intel_sdvo_connector);
 }
 
-static bool intel_sdvo_detect_hdmi_audio(struct drm_connector *connector)
-{
-	struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
-	struct edid *edid;
-	bool has_audio = false;
-
-	if (!intel_sdvo->is_hdmi)
-		return false;
-
-	edid = intel_sdvo_get_edid(connector);
-	if (edid != NULL && edid->input & DRM_EDID_INPUT_DIGITAL)
-		has_audio = drm_detect_monitor_audio(edid);
-	kfree(edid);
-
-	return has_audio;
-}
-
 static int
-intel_sdvo_set_property(struct drm_connector *connector,
-			struct drm_property *property,
-			uint64_t val)
+intel_sdvo_connector_atomic_get_property(struct drm_connector *connector,
+					 const struct drm_connector_state *state,
+					 struct drm_property *property,
+					 uint64_t *val)
 {
-	struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
 	struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	uint16_t temp_value;
-	uint8_t cmd;
-	int ret;
-
-	ret = drm_object_property_set_value(&connector->base, property, val);
-	if (ret)
-		return ret;
-
-	if (property == dev_priv->force_audio_property) {
-		int i = val;
-		bool has_audio;
-
-		if (i == intel_sdvo_connector->force_audio)
-			return 0;
-
-		intel_sdvo_connector->force_audio = i;
-
-		if (i == HDMI_AUDIO_AUTO)
-			has_audio = intel_sdvo_detect_hdmi_audio(connector);
-		else
-			has_audio = (i == HDMI_AUDIO_ON);
-
-		if (has_audio == intel_sdvo->has_hdmi_audio)
-			return 0;
-
-		intel_sdvo->has_hdmi_audio = has_audio;
-		goto done;
-	}
-
-	if (property == dev_priv->broadcast_rgb_property) {
-		bool old_auto = intel_sdvo->color_range_auto;
-		uint32_t old_range = intel_sdvo->color_range;
-
-		switch (val) {
-		case INTEL_BROADCAST_RGB_AUTO:
-			intel_sdvo->color_range_auto = true;
-			break;
-		case INTEL_BROADCAST_RGB_FULL:
-			intel_sdvo->color_range_auto = false;
-			intel_sdvo->color_range = 0;
-			break;
-		case INTEL_BROADCAST_RGB_LIMITED:
-			intel_sdvo->color_range_auto = false;
-			/* FIXME: this bit is only valid when using TMDS
-			 * encoding and 8 bit per color mode. */
-			intel_sdvo->color_range = HDMI_COLOR_RANGE_16_235;
-			break;
-		default:
-			return -EINVAL;
-		}
-
-		if (old_auto == intel_sdvo->color_range_auto &&
-		    old_range == intel_sdvo->color_range)
-			return 0;
-
-		goto done;
-	}
-
-	if (property == connector->dev->mode_config.aspect_ratio_property) {
-		switch (val) {
-		case DRM_MODE_PICTURE_ASPECT_NONE:
-			intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
-			break;
-		case DRM_MODE_PICTURE_ASPECT_4_3:
-			intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_4_3;
-			break;
-		case DRM_MODE_PICTURE_ASPECT_16_9:
-			intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_16_9;
-			break;
-		default:
-			return -EINVAL;
-		}
-		goto done;
-	}
-
-#define CHECK_PROPERTY(name, NAME) \
-	if (intel_sdvo_connector->name == property) { \
-		if (intel_sdvo_connector->cur_##name == temp_value) return 0; \
-		if (intel_sdvo_connector->max_##name < temp_value) return -EINVAL; \
-		cmd = SDVO_CMD_SET_##NAME; \
-		intel_sdvo_connector->cur_##name = temp_value; \
-		goto set_value; \
-	}
+	const struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state((void *)state);
 
 	if (property == intel_sdvo_connector->tv_format) {
-		if (val >= TV_FORMAT_NUM)
-			return -EINVAL;
-
-		if (intel_sdvo->tv_format_index ==
-		    intel_sdvo_connector->tv_format_supported[val])
-			return 0;
-
-		intel_sdvo->tv_format_index = intel_sdvo_connector->tv_format_supported[val];
-		goto done;
-	} else if (IS_TV_OR_LVDS(intel_sdvo_connector)) {
-		temp_value = val;
-		if (intel_sdvo_connector->left == property) {
-			drm_object_property_set_value(&connector->base,
-							 intel_sdvo_connector->right, val);
-			if (intel_sdvo_connector->left_margin == temp_value)
-				return 0;
+		int i;
 
-			intel_sdvo_connector->left_margin = temp_value;
-			intel_sdvo_connector->right_margin = temp_value;
-			temp_value = intel_sdvo_connector->max_hscan -
-				intel_sdvo_connector->left_margin;
-			cmd = SDVO_CMD_SET_OVERSCAN_H;
-			goto set_value;
-		} else if (intel_sdvo_connector->right == property) {
-			drm_object_property_set_value(&connector->base,
-							 intel_sdvo_connector->left, val);
-			if (intel_sdvo_connector->right_margin == temp_value)
-				return 0;
+		for (i = 0; i < intel_sdvo_connector->format_supported_num; i++)
+			if (state->tv.mode == intel_sdvo_connector->tv_format_supported[i]) {
+				*val = i;
 
-			intel_sdvo_connector->left_margin = temp_value;
-			intel_sdvo_connector->right_margin = temp_value;
-			temp_value = intel_sdvo_connector->max_hscan -
-				intel_sdvo_connector->left_margin;
-			cmd = SDVO_CMD_SET_OVERSCAN_H;
-			goto set_value;
-		} else if (intel_sdvo_connector->top == property) {
-			drm_object_property_set_value(&connector->base,
-							 intel_sdvo_connector->bottom, val);
-			if (intel_sdvo_connector->top_margin == temp_value)
 				return 0;
+			}
 
-			intel_sdvo_connector->top_margin = temp_value;
-			intel_sdvo_connector->bottom_margin = temp_value;
-			temp_value = intel_sdvo_connector->max_vscan -
-				intel_sdvo_connector->top_margin;
-			cmd = SDVO_CMD_SET_OVERSCAN_V;
-			goto set_value;
-		} else if (intel_sdvo_connector->bottom == property) {
-			drm_object_property_set_value(&connector->base,
-							 intel_sdvo_connector->top, val);
-			if (intel_sdvo_connector->bottom_margin == temp_value)
-				return 0;
+		WARN_ON(1);
+		*val = 0;
+	} else if (property == intel_sdvo_connector->top ||
+		   property == intel_sdvo_connector->bottom)
+		*val = intel_sdvo_connector->max_vscan - sdvo_state->tv.overscan_v;
+	else if (property == intel_sdvo_connector->left ||
+		 property == intel_sdvo_connector->right)
+		*val = intel_sdvo_connector->max_hscan - sdvo_state->tv.overscan_h;
+	else if (property == intel_sdvo_connector->hpos)
+		*val = sdvo_state->tv.hpos;
+	else if (property == intel_sdvo_connector->vpos)
+		*val = sdvo_state->tv.vpos;
+	else if (property == intel_sdvo_connector->saturation)
+		*val = state->tv.saturation;
+	else if (property == intel_sdvo_connector->contrast)
+		*val = state->tv.contrast;
+	else if (property == intel_sdvo_connector->hue)
+		*val = state->tv.hue;
+	else if (property == intel_sdvo_connector->brightness)
+		*val = state->tv.brightness;
+	else if (property == intel_sdvo_connector->sharpness)
+		*val = sdvo_state->tv.sharpness;
+	else if (property == intel_sdvo_connector->flicker_filter)
+		*val = sdvo_state->tv.flicker_filter;
+	else if (property == intel_sdvo_connector->flicker_filter_2d)
+		*val = sdvo_state->tv.flicker_filter_2d;
+	else if (property == intel_sdvo_connector->flicker_filter_adaptive)
+		*val = sdvo_state->tv.flicker_filter_adaptive;
+	else if (property == intel_sdvo_connector->tv_chroma_filter)
+		*val = sdvo_state->tv.chroma_filter;
+	else if (property == intel_sdvo_connector->tv_luma_filter)
+		*val = sdvo_state->tv.luma_filter;
+	else if (property == intel_sdvo_connector->dot_crawl)
+		*val = sdvo_state->tv.dot_crawl;
+	else
+		return intel_digital_connector_atomic_get_property(connector, state, property, val);
 
-			intel_sdvo_connector->top_margin = temp_value;
-			intel_sdvo_connector->bottom_margin = temp_value;
-			temp_value = intel_sdvo_connector->max_vscan -
-				intel_sdvo_connector->top_margin;
-			cmd = SDVO_CMD_SET_OVERSCAN_V;
-			goto set_value;
-		}
-		CHECK_PROPERTY(hpos, HPOS)
-		CHECK_PROPERTY(vpos, VPOS)
-		CHECK_PROPERTY(saturation, SATURATION)
-		CHECK_PROPERTY(contrast, CONTRAST)
-		CHECK_PROPERTY(hue, HUE)
-		CHECK_PROPERTY(brightness, BRIGHTNESS)
-		CHECK_PROPERTY(sharpness, SHARPNESS)
-		CHECK_PROPERTY(flicker_filter, FLICKER_FILTER)
-		CHECK_PROPERTY(flicker_filter_2d, FLICKER_FILTER_2D)
-		CHECK_PROPERTY(flicker_filter_adaptive, FLICKER_FILTER_ADAPTIVE)
-		CHECK_PROPERTY(tv_chroma_filter, TV_CHROMA_FILTER)
-		CHECK_PROPERTY(tv_luma_filter, TV_LUMA_FILTER)
-		CHECK_PROPERTY(dot_crawl, DOT_CRAWL)
-	}
+	return 0;
+}
 
-	return -EINVAL; /* unknown property */
+static int
+intel_sdvo_connector_atomic_set_property(struct drm_connector *connector,
+					 struct drm_connector_state *state,
+					 struct drm_property *property,
+					 uint64_t val)
+{
+	struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
+	struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state(state);
 
-set_value:
-	if (!intel_sdvo_set_value(intel_sdvo, cmd, &temp_value, 2))
-		return -EIO;
+	if (property == intel_sdvo_connector->tv_format) {
+		state->tv.mode = intel_sdvo_connector->tv_format_supported[val];
 
+		if (state->crtc) {
+			struct drm_crtc_state *crtc_state =
+				drm_atomic_get_new_crtc_state(state->state, state->crtc);
 
-done:
-	if (intel_sdvo->base.base.crtc)
-		intel_crtc_restore_mode(intel_sdvo->base.base.crtc);
+			crtc_state->connectors_changed = true;
+		}
+	} else if (property == intel_sdvo_connector->top ||
+		   property == intel_sdvo_connector->bottom)
+		/* Cannot set these independent from each other */
+		sdvo_state->tv.overscan_v = intel_sdvo_connector->max_vscan - val;
+	else if (property == intel_sdvo_connector->left ||
+		 property == intel_sdvo_connector->right)
+		/* Cannot set these independent from each other */
+		sdvo_state->tv.overscan_h = intel_sdvo_connector->max_hscan - val;
+	else if (property == intel_sdvo_connector->hpos)
+		sdvo_state->tv.hpos = val;
+	else if (property == intel_sdvo_connector->vpos)
+		sdvo_state->tv.vpos = val;
+	else if (property == intel_sdvo_connector->saturation)
+		state->tv.saturation = val;
+	else if (property == intel_sdvo_connector->contrast)
+		state->tv.contrast = val;
+	else if (property == intel_sdvo_connector->hue)
+		state->tv.hue = val;
+	else if (property == intel_sdvo_connector->brightness)
+		state->tv.brightness = val;
+	else if (property == intel_sdvo_connector->sharpness)
+		sdvo_state->tv.sharpness = val;
+	else if (property == intel_sdvo_connector->flicker_filter)
+		sdvo_state->tv.flicker_filter = val;
+	else if (property == intel_sdvo_connector->flicker_filter_2d)
+		sdvo_state->tv.flicker_filter_2d = val;
+	else if (property == intel_sdvo_connector->flicker_filter_adaptive)
+		sdvo_state->tv.flicker_filter_adaptive = val;
+	else if (property == intel_sdvo_connector->tv_chroma_filter)
+		sdvo_state->tv.chroma_filter = val;
+	else if (property == intel_sdvo_connector->tv_luma_filter)
+		sdvo_state->tv.luma_filter = val;
+	else if (property == intel_sdvo_connector->dot_crawl)
+		sdvo_state->tv.dot_crawl = val;
+	else
+		return intel_digital_connector_atomic_set_property(connector, state, property, val);
 
 	return 0;
-#undef CHECK_PROPERTY
 }
 
 static int
@@ -2208,22 +2178,61 @@ intel_sdvo_connector_unregister(struct drm_connector *connector)
 	intel_connector_unregister(connector);
 }
 
+static struct drm_connector_state *
+intel_sdvo_connector_duplicate_state(struct drm_connector *connector)
+{
+	struct intel_sdvo_connector_state *state;
+
+	state = kmemdup(connector->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_connector_duplicate_state(connector, &state->base.base);
+	return &state->base.base;
+}
+
 static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_sdvo_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = intel_sdvo_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.atomic_get_property = intel_sdvo_connector_atomic_get_property,
+	.atomic_set_property = intel_sdvo_connector_atomic_set_property,
 	.late_register = intel_sdvo_connector_register,
 	.early_unregister = intel_sdvo_connector_unregister,
 	.destroy = intel_sdvo_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_duplicate_state = intel_sdvo_connector_duplicate_state,
 };
 
+static int intel_sdvo_atomic_check(struct drm_connector *conn,
+				   struct drm_connector_state *new_conn_state)
+{
+	struct drm_atomic_state *state = new_conn_state->state;
+	struct drm_connector_state *old_conn_state =
+		drm_atomic_get_old_connector_state(state, conn);
+	struct intel_sdvo_connector_state *old_state =
+		to_intel_sdvo_connector_state(old_conn_state);
+	struct intel_sdvo_connector_state *new_state =
+		to_intel_sdvo_connector_state(new_conn_state);
+
+	if (new_conn_state->crtc &&
+	    (memcmp(&old_state->tv, &new_state->tv, sizeof(old_state->tv)) ||
+	     memcmp(&old_conn_state->tv, &new_conn_state->tv, sizeof(old_conn_state->tv)))) {
+		struct drm_crtc_state *crtc_state =
+			drm_atomic_get_new_crtc_state(new_conn_state->state,
+						      new_conn_state->crtc);
+
+		crtc_state->connectors_changed = true;
+	}
+
+	return intel_digital_connector_atomic_check(conn, new_conn_state);
+}
+
 static const struct drm_connector_helper_funcs intel_sdvo_connector_helper_funcs = {
 	.get_modes = intel_sdvo_get_modes,
 	.mode_valid = intel_sdvo_mode_valid,
+	.atomic_check = intel_sdvo_atomic_check,
 };
 
 static void intel_sdvo_enc_destroy(struct drm_encoder *encoder)
@@ -2415,25 +2424,29 @@ intel_sdvo_add_hdmi_properties(struct intel_sdvo *intel_sdvo,
 	intel_attach_force_audio_property(&connector->base.base);
 	if (INTEL_GEN(dev_priv) >= 4 && IS_MOBILE(dev_priv)) {
 		intel_attach_broadcast_rgb_property(&connector->base.base);
-		intel_sdvo->color_range_auto = true;
 	}
 	intel_attach_aspect_ratio_property(&connector->base.base);
-	intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
+	connector->base.base.state->picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
 }
 
 static struct intel_sdvo_connector *intel_sdvo_connector_alloc(void)
 {
 	struct intel_sdvo_connector *sdvo_connector;
+	struct intel_sdvo_connector_state *conn_state;
 
 	sdvo_connector = kzalloc(sizeof(*sdvo_connector), GFP_KERNEL);
 	if (!sdvo_connector)
 		return NULL;
 
-	if (intel_connector_init(&sdvo_connector->base) < 0) {
+	conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL);
+	if (!conn_state) {
 		kfree(sdvo_connector);
 		return NULL;
 	}
 
+	__drm_atomic_helper_connector_reset(&sdvo_connector->base.base,
+					    &conn_state->base.base);
+
 	return sdvo_connector;
 }
 
@@ -2725,31 +2738,31 @@ static bool intel_sdvo_tv_create_property(struct intel_sdvo *intel_sdvo,
 				intel_sdvo_connector->tv_format, i,
 				i, tv_format_names[intel_sdvo_connector->tv_format_supported[i]]);
 
-	intel_sdvo->tv_format_index = intel_sdvo_connector->tv_format_supported[0];
+	intel_sdvo_connector->base.base.state->tv.mode = intel_sdvo_connector->tv_format_supported[0];
 	drm_object_attach_property(&intel_sdvo_connector->base.base.base,
-				      intel_sdvo_connector->tv_format, 0);
+				   intel_sdvo_connector->tv_format, 0);
 	return true;
 
 }
 
-#define ENHANCEMENT(name, NAME) do { \
+#define _ENHANCEMENT(state_assignment, name, NAME) do { \
 	if (enhancements.name) { \
 		if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_MAX_##NAME, &data_value, 4) || \
 		    !intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_##NAME, &response, 2)) \
 			return false; \
-		intel_sdvo_connector->max_##name = data_value[0]; \
-		intel_sdvo_connector->cur_##name = response; \
 		intel_sdvo_connector->name = \
 			drm_property_create_range(dev, 0, #name, 0, data_value[0]); \
 		if (!intel_sdvo_connector->name) return false; \
+		state_assignment = response; \
 		drm_object_attach_property(&connector->base, \
-					      intel_sdvo_connector->name, \
-					      intel_sdvo_connector->cur_##name); \
+					   intel_sdvo_connector->name, 0); \
 		DRM_DEBUG_KMS(#name ": max %d, default %d, current %d\n", \
 			      data_value[0], data_value[1], response); \
 	} \
 } while (0)
 
+#define ENHANCEMENT(state, name, NAME) _ENHANCEMENT((state)->name, name, NAME)
+
 static bool
 intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 				      struct intel_sdvo_connector *intel_sdvo_connector,
@@ -2757,6 +2770,9 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 {
 	struct drm_device *dev = intel_sdvo->base.base.dev;
 	struct drm_connector *connector = &intel_sdvo_connector->base.base;
+	struct drm_connector_state *conn_state = connector->state;
+	struct intel_sdvo_connector_state *sdvo_state =
+		to_intel_sdvo_connector_state(conn_state);
 	uint16_t response, data_value[2];
 
 	/* when horizontal overscan is supported, Add the left/right  property */
@@ -2771,17 +2787,16 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 					  &response, 2))
 			return false;
 
+		sdvo_state->tv.overscan_h = response;
+
 		intel_sdvo_connector->max_hscan = data_value[0];
-		intel_sdvo_connector->left_margin = data_value[0] - response;
-		intel_sdvo_connector->right_margin = intel_sdvo_connector->left_margin;
 		intel_sdvo_connector->left =
 			drm_property_create_range(dev, 0, "left_margin", 0, data_value[0]);
 		if (!intel_sdvo_connector->left)
 			return false;
 
 		drm_object_attach_property(&connector->base,
-					      intel_sdvo_connector->left,
-					      intel_sdvo_connector->left_margin);
+					   intel_sdvo_connector->left, 0);
 
 		intel_sdvo_connector->right =
 			drm_property_create_range(dev, 0, "right_margin", 0, data_value[0]);
@@ -2789,8 +2804,7 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 			return false;
 
 		drm_object_attach_property(&connector->base,
-					      intel_sdvo_connector->right,
-					      intel_sdvo_connector->right_margin);
+					      intel_sdvo_connector->right, 0);
 		DRM_DEBUG_KMS("h_overscan: max %d, "
 			      "default %d, current %d\n",
 			      data_value[0], data_value[1], response);
@@ -2807,9 +2821,9 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 					  &response, 2))
 			return false;
 
+		sdvo_state->tv.overscan_v = response;
+
 		intel_sdvo_connector->max_vscan = data_value[0];
-		intel_sdvo_connector->top_margin = data_value[0] - response;
-		intel_sdvo_connector->bottom_margin = intel_sdvo_connector->top_margin;
 		intel_sdvo_connector->top =
 			drm_property_create_range(dev, 0,
 					    "top_margin", 0, data_value[0]);
@@ -2817,8 +2831,7 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 			return false;
 
 		drm_object_attach_property(&connector->base,
-					      intel_sdvo_connector->top,
-					      intel_sdvo_connector->top_margin);
+					   intel_sdvo_connector->top, 0);
 
 		intel_sdvo_connector->bottom =
 			drm_property_create_range(dev, 0,
@@ -2827,40 +2840,37 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
 			return false;
 
 		drm_object_attach_property(&connector->base,
-					      intel_sdvo_connector->bottom,
-					      intel_sdvo_connector->bottom_margin);
+					      intel_sdvo_connector->bottom, 0);
 		DRM_DEBUG_KMS("v_overscan: max %d, "
 			      "default %d, current %d\n",
 			      data_value[0], data_value[1], response);
 	}
 
-	ENHANCEMENT(hpos, HPOS);
-	ENHANCEMENT(vpos, VPOS);
-	ENHANCEMENT(saturation, SATURATION);
-	ENHANCEMENT(contrast, CONTRAST);
-	ENHANCEMENT(hue, HUE);
-	ENHANCEMENT(sharpness, SHARPNESS);
-	ENHANCEMENT(brightness, BRIGHTNESS);
-	ENHANCEMENT(flicker_filter, FLICKER_FILTER);
-	ENHANCEMENT(flicker_filter_adaptive, FLICKER_FILTER_ADAPTIVE);
-	ENHANCEMENT(flicker_filter_2d, FLICKER_FILTER_2D);
-	ENHANCEMENT(tv_chroma_filter, TV_CHROMA_FILTER);
-	ENHANCEMENT(tv_luma_filter, TV_LUMA_FILTER);
+	ENHANCEMENT(&sdvo_state->tv, hpos, HPOS);
+	ENHANCEMENT(&sdvo_state->tv, vpos, VPOS);
+	ENHANCEMENT(&conn_state->tv, saturation, SATURATION);
+	ENHANCEMENT(&conn_state->tv, contrast, CONTRAST);
+	ENHANCEMENT(&conn_state->tv, hue, HUE);
+	ENHANCEMENT(&conn_state->tv, brightness, BRIGHTNESS);
+	ENHANCEMENT(&sdvo_state->tv, sharpness, SHARPNESS);
+	ENHANCEMENT(&sdvo_state->tv, flicker_filter, FLICKER_FILTER);
+	ENHANCEMENT(&sdvo_state->tv, flicker_filter_adaptive, FLICKER_FILTER_ADAPTIVE);
+	ENHANCEMENT(&sdvo_state->tv, flicker_filter_2d, FLICKER_FILTER_2D);
+	_ENHANCEMENT(sdvo_state->tv.chroma_filter, tv_chroma_filter, TV_CHROMA_FILTER);
+	_ENHANCEMENT(sdvo_state->tv.luma_filter, tv_luma_filter, TV_LUMA_FILTER);
 
 	if (enhancements.dot_crawl) {
 		if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_DOT_CRAWL, &response, 2))
 			return false;
 
-		intel_sdvo_connector->max_dot_crawl = 1;
-		intel_sdvo_connector->cur_dot_crawl = response & 0x1;
+		sdvo_state->tv.dot_crawl = response & 0x1;
 		intel_sdvo_connector->dot_crawl =
 			drm_property_create_range(dev, 0, "dot_crawl", 0, 1);
 		if (!intel_sdvo_connector->dot_crawl)
 			return false;
 
 		drm_object_attach_property(&connector->base,
-					      intel_sdvo_connector->dot_crawl,
-					      intel_sdvo_connector->cur_dot_crawl);
+					   intel_sdvo_connector->dot_crawl, 0);
 		DRM_DEBUG_KMS("dot crawl: current %d\n", response);
 	}
 
@@ -2876,11 +2886,12 @@ intel_sdvo_create_enhance_property_lvds(struct intel_sdvo *intel_sdvo,
 	struct drm_connector *connector = &intel_sdvo_connector->base.base;
 	uint16_t response, data_value[2];
 
-	ENHANCEMENT(brightness, BRIGHTNESS);
+	ENHANCEMENT(&connector->state->tv, brightness, BRIGHTNESS);
 
 	return true;
 }
 #undef ENHANCEMENT
+#undef _ENHANCEMENT
 
 static bool intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo,
 					       struct intel_sdvo_connector *intel_sdvo_connector)
@@ -2892,11 +2903,10 @@ static bool intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo,
 
 	BUILD_BUG_ON(sizeof(enhancements) != 2);
 
-	enhancements.response = 0;
-	intel_sdvo_get_value(intel_sdvo,
-			     SDVO_CMD_GET_SUPPORTED_ENHANCEMENTS,
-			     &enhancements, sizeof(enhancements));
-	if (enhancements.response == 0) {
+	if (!intel_sdvo_get_value(intel_sdvo,
+				  SDVO_CMD_GET_SUPPORTED_ENHANCEMENTS,
+				  &enhancements, sizeof(enhancements)) ||
+	    enhancements.response == 0) {
 		DRM_DEBUG_KMS("No enhancement is supported\n");
 		return true;
 	}
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index e6517edcd16b..0c650c2cbca8 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -231,16 +231,14 @@ void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work
 }
 
 static void
-skl_update_plane(struct drm_plane *drm_plane,
+skl_update_plane(struct intel_plane *plane,
 		 const struct intel_crtc_state *crtc_state,
 		 const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = drm_plane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(drm_plane);
-	struct drm_framebuffer *fb = plane_state->base.fb;
-	enum plane_id plane_id = intel_plane->id;
-	enum pipe pipe = intel_plane->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum plane_id plane_id = plane->id;
+	enum pipe pipe = plane->pipe;
 	u32 plane_ctl = plane_state->ctl;
 	const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
 	u32 surf_addr = plane_state->main.offset;
@@ -309,13 +307,11 @@ skl_update_plane(struct drm_plane *drm_plane,
 }
 
 static void
-skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
+skl_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc)
 {
-	struct drm_device *dev = dplane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(dplane);
-	enum plane_id plane_id = intel_plane->id;
-	enum pipe pipe = intel_plane->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum plane_id plane_id = plane->id;
+	enum pipe pipe = plane->pipe;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
@@ -329,10 +325,10 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 }
 
 static void
-chv_update_csc(struct intel_plane *intel_plane, uint32_t format)
+chv_update_csc(struct intel_plane *plane, uint32_t format)
 {
-	struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
-	enum plane_id plane_id = intel_plane->id;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum plane_id plane_id = plane->id;
 
 	/* Seems RGB data bypasses the CSC always */
 	if (!format_is_yuv(format))
@@ -419,10 +415,10 @@ static u32 vlv_sprite_ctl(const struct intel_crtc_state *crtc_state,
 	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		sprctl |= SP_TILED;
 
-	if (rotation & DRM_ROTATE_180)
+	if (rotation & DRM_MODE_ROTATE_180)
 		sprctl |= SP_ROTATE_180;
 
-	if (rotation & DRM_REFLECT_X)
+	if (rotation & DRM_MODE_REFLECT_X)
 		sprctl |= SP_MIRROR;
 
 	if (key->flags & I915_SET_COLORKEY_SOURCE)
@@ -432,16 +428,14 @@ static u32 vlv_sprite_ctl(const struct intel_crtc_state *crtc_state,
 }
 
 static void
-vlv_update_plane(struct drm_plane *dplane,
+vlv_update_plane(struct intel_plane *plane,
 		 const struct intel_crtc_state *crtc_state,
 		 const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = dplane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(dplane);
-	struct drm_framebuffer *fb = plane_state->base.fb;
-	enum pipe pipe = intel_plane->pipe;
-	enum plane_id plane_id = intel_plane->id;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum pipe pipe = plane->pipe;
+	enum plane_id plane_id = plane->id;
 	u32 sprctl = plane_state->ctl;
 	u32 sprsurf_offset = plane_state->main.offset;
 	u32 linear_offset;
@@ -463,7 +457,7 @@ vlv_update_plane(struct drm_plane *dplane,
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
 	if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B)
-		chv_update_csc(intel_plane, fb->format->format);
+		chv_update_csc(plane, fb->format->format);
 
 	if (key->flags) {
 		I915_WRITE_FW(SPKEYMINVAL(pipe, plane_id), key->min_value);
@@ -490,13 +484,11 @@ vlv_update_plane(struct drm_plane *dplane,
 }
 
 static void
-vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
+vlv_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc)
 {
-	struct drm_device *dev = dplane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(dplane);
-	enum pipe pipe = intel_plane->pipe;
-	enum plane_id plane_id = intel_plane->id;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum pipe pipe = plane->pipe;
+	enum plane_id plane_id = plane->id;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
@@ -554,7 +546,7 @@ static u32 ivb_sprite_ctl(const struct intel_crtc_state *crtc_state,
 	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		sprctl |= SPRITE_TILED;
 
-	if (rotation & DRM_ROTATE_180)
+	if (rotation & DRM_MODE_ROTATE_180)
 		sprctl |= SPRITE_ROTATE_180;
 
 	if (key->flags & I915_SET_COLORKEY_DESTINATION)
@@ -566,15 +558,13 @@ static u32 ivb_sprite_ctl(const struct intel_crtc_state *crtc_state,
 }
 
 static void
-ivb_update_plane(struct drm_plane *plane,
+ivb_update_plane(struct intel_plane *plane,
 		 const struct intel_crtc_state *crtc_state,
 		 const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = plane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(plane);
-	struct drm_framebuffer *fb = plane_state->base.fb;
-	enum pipe pipe = intel_plane->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum pipe pipe = plane->pipe;
 	u32 sprctl = plane_state->ctl, sprscale = 0;
 	u32 sprsurf_offset = plane_state->main.offset;
 	u32 linear_offset;
@@ -621,7 +611,7 @@ ivb_update_plane(struct drm_plane *plane,
 		I915_WRITE_FW(SPRLINOFF(pipe), linear_offset);
 
 	I915_WRITE_FW(SPRSIZE(pipe), (crtc_h << 16) | crtc_w);
-	if (intel_plane->can_scale)
+	if (plane->can_scale)
 		I915_WRITE_FW(SPRSCALE(pipe), sprscale);
 	I915_WRITE_FW(SPRCTL(pipe), sprctl);
 	I915_WRITE_FW(SPRSURF(pipe),
@@ -632,19 +622,17 @@ ivb_update_plane(struct drm_plane *plane,
 }
 
 static void
-ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc)
+ivb_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc)
 {
-	struct drm_device *dev = plane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(plane);
-	int pipe = intel_plane->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum pipe pipe = plane->pipe;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
 	I915_WRITE_FW(SPRCTL(pipe), 0);
 	/* Can't leave the scaler enabled... */
-	if (intel_plane->can_scale)
+	if (plane->can_scale)
 		I915_WRITE_FW(SPRSCALE(pipe), 0);
 
 	I915_WRITE_FW(SPRSURF(pipe), 0);
@@ -653,7 +641,7 @@ ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc)
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static u32 ilk_sprite_ctl(const struct intel_crtc_state *crtc_state,
+static u32 g4x_sprite_ctl(const struct intel_crtc_state *crtc_state,
 			  const struct intel_plane_state *plane_state)
 {
 	struct drm_i915_private *dev_priv =
@@ -695,7 +683,7 @@ static u32 ilk_sprite_ctl(const struct intel_crtc_state *crtc_state,
 	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		dvscntr |= DVS_TILED;
 
-	if (rotation & DRM_ROTATE_180)
+	if (rotation & DRM_MODE_ROTATE_180)
 		dvscntr |= DVS_ROTATE_180;
 
 	if (key->flags & I915_SET_COLORKEY_DESTINATION)
@@ -707,15 +695,13 @@ static u32 ilk_sprite_ctl(const struct intel_crtc_state *crtc_state,
 }
 
 static void
-ilk_update_plane(struct drm_plane *plane,
+g4x_update_plane(struct intel_plane *plane,
 		 const struct intel_crtc_state *crtc_state,
 		 const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = plane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(plane);
-	struct drm_framebuffer *fb = plane_state->base.fb;
-	int pipe = intel_plane->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	enum pipe pipe = plane->pipe;
 	u32 dvscntr = plane_state->ctl, dvsscale = 0;
 	u32 dvssurf_offset = plane_state->main.offset;
 	u32 linear_offset;
@@ -768,12 +754,10 @@ ilk_update_plane(struct drm_plane *plane,
 }
 
 static void
-ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc)
+g4x_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc)
 {
-	struct drm_device *dev = plane->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_plane *intel_plane = to_intel_plane(plane);
-	int pipe = intel_plane->pipe;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum pipe pipe = plane->pipe;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
@@ -789,14 +773,12 @@ ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc)
 }
 
 static int
-intel_check_sprite_plane(struct drm_plane *plane,
+intel_check_sprite_plane(struct intel_plane *plane,
 			 struct intel_crtc_state *crtc_state,
 			 struct intel_plane_state *state)
 {
-	struct drm_i915_private *dev_priv = to_i915(plane->dev);
-	struct drm_crtc *crtc = state->base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_plane *intel_plane = to_intel_plane(plane);
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_framebuffer *fb = state->base.fb;
 	int crtc_x, crtc_y;
 	unsigned int crtc_w, crtc_h;
@@ -818,7 +800,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	}
 
 	/* Don't modify another pipe's plane */
-	if (intel_plane->pipe != intel_crtc->pipe) {
+	if (plane->pipe != crtc->pipe) {
 		DRM_DEBUG_KMS("Wrong plane <-> crtc mapping\n");
 		return -EINVAL;
 	}
@@ -835,16 +817,16 @@ intel_check_sprite_plane(struct drm_plane *plane,
 		if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
 			can_scale = 1;
 			min_scale = 1;
-			max_scale = skl_max_scale(intel_crtc, crtc_state);
+			max_scale = skl_max_scale(crtc, crtc_state);
 		} else {
 			can_scale = 0;
 			min_scale = DRM_PLANE_HELPER_NO_SCALING;
 			max_scale = DRM_PLANE_HELPER_NO_SCALING;
 		}
 	} else {
-		can_scale = intel_plane->can_scale;
-		max_scale = intel_plane->max_downscale << 16;
-		min_scale = intel_plane->can_scale ? 1 : (1 << 16);
+		can_scale = plane->can_scale;
+		max_scale = plane->max_downscale << 16;
+		min_scale = plane->can_scale ? 1 : (1 << 16);
 	}
 
 	/*
@@ -988,7 +970,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 		if (ret)
 			return ret;
 
-		state->ctl = ilk_sprite_ctl(crtc_state, state);
+		state->ctl = g4x_sprite_ctl(crtc_state, state);
 	}
 
 	return 0;
@@ -1048,7 +1030,7 @@ out:
 	return ret;
 }
 
-static const uint32_t ilk_plane_formats[] = {
+static const uint32_t g4x_plane_formats[] = {
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_YUYV,
 	DRM_FORMAT_YVYU,
@@ -1152,29 +1134,29 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 		intel_plane->can_scale = true;
 		intel_plane->max_downscale = 16;
 
-		intel_plane->update_plane = ilk_update_plane;
-		intel_plane->disable_plane = ilk_disable_plane;
+		intel_plane->update_plane = g4x_update_plane;
+		intel_plane->disable_plane = g4x_disable_plane;
 
 		if (IS_GEN6(dev_priv)) {
 			plane_formats = snb_plane_formats;
 			num_plane_formats = ARRAY_SIZE(snb_plane_formats);
 		} else {
-			plane_formats = ilk_plane_formats;
-			num_plane_formats = ARRAY_SIZE(ilk_plane_formats);
+			plane_formats = g4x_plane_formats;
+			num_plane_formats = ARRAY_SIZE(g4x_plane_formats);
 		}
 	}
 
 	if (INTEL_GEN(dev_priv) >= 9) {
 		supported_rotations =
-			DRM_ROTATE_0 | DRM_ROTATE_90 |
-			DRM_ROTATE_180 | DRM_ROTATE_270;
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+			DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270;
 	} else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
 		supported_rotations =
-			DRM_ROTATE_0 | DRM_ROTATE_180 |
-			DRM_REFLECT_X;
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
+			DRM_MODE_REFLECT_X;
 	} else {
 		supported_rotations =
-			DRM_ROTATE_0 | DRM_ROTATE_180;
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
 	}
 
 	intel_plane->pipe = pipe;
@@ -1201,7 +1183,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 		goto fail;
 
 	drm_plane_create_rotation_property(&intel_plane->base,
-					   DRM_ROTATE_0,
+					   DRM_MODE_ROTATE_0,
 					   supported_rotations);
 
 	drm_plane_helper_add(&intel_plane->base, &intel_plane_helper_funcs);
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index e077c2a9e694..784df024e230 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -48,41 +48,6 @@ struct intel_tv {
 	struct intel_encoder base;
 
 	int type;
-	const char *tv_format;
-	int margin[4];
-	u32 save_TV_H_CTL_1;
-	u32 save_TV_H_CTL_2;
-	u32 save_TV_H_CTL_3;
-	u32 save_TV_V_CTL_1;
-	u32 save_TV_V_CTL_2;
-	u32 save_TV_V_CTL_3;
-	u32 save_TV_V_CTL_4;
-	u32 save_TV_V_CTL_5;
-	u32 save_TV_V_CTL_6;
-	u32 save_TV_V_CTL_7;
-	u32 save_TV_SC_CTL_1, save_TV_SC_CTL_2, save_TV_SC_CTL_3;
-
-	u32 save_TV_CSC_Y;
-	u32 save_TV_CSC_Y2;
-	u32 save_TV_CSC_U;
-	u32 save_TV_CSC_U2;
-	u32 save_TV_CSC_V;
-	u32 save_TV_CSC_V2;
-	u32 save_TV_CLR_KNOBS;
-	u32 save_TV_CLR_LEVEL;
-	u32 save_TV_WIN_POS;
-	u32 save_TV_WIN_SIZE;
-	u32 save_TV_FILTER_CTL_1;
-	u32 save_TV_FILTER_CTL_2;
-	u32 save_TV_FILTER_CTL_3;
-
-	u32 save_TV_H_LUMA[60];
-	u32 save_TV_H_CHROMA[60];
-	u32 save_TV_V_LUMA[43];
-	u32 save_TV_V_CHROMA[43];
-
-	u32 save_TV_DAC;
-	u32 save_TV_CTL;
 };
 
 struct video_levels {
@@ -873,32 +838,18 @@ intel_disable_tv(struct intel_encoder *encoder,
 	I915_WRITE(TV_CTL, I915_READ(TV_CTL) & ~TV_ENC_ENABLE);
 }
 
-static const struct tv_mode *
-intel_tv_mode_lookup(const char *tv_format)
+static const struct tv_mode *intel_tv_mode_find(struct drm_connector_state *conn_state)
 {
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(tv_modes); i++) {
-		const struct tv_mode *tv_mode = &tv_modes[i];
+	int format = conn_state->tv.mode;
 
-		if (!strcmp(tv_format, tv_mode->name))
-			return tv_mode;
-	}
-	return NULL;
-}
-
-static const struct tv_mode *
-intel_tv_mode_find(struct intel_tv *intel_tv)
-{
-	return intel_tv_mode_lookup(intel_tv->tv_format);
+	return &tv_modes[format];
 }
 
 static enum drm_mode_status
 intel_tv_mode_valid(struct drm_connector *connector,
 		    struct drm_display_mode *mode)
 {
-	struct intel_tv *intel_tv = intel_attached_tv(connector);
-	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
+	const struct tv_mode *tv_mode = intel_tv_mode_find(connector->state);
 	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
 
 	if (mode->clock > max_dotclk)
@@ -925,8 +876,7 @@ intel_tv_compute_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
 			struct drm_connector_state *conn_state)
 {
-	struct intel_tv *intel_tv = enc_to_tv(encoder);
-	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
+	const struct tv_mode *tv_mode = intel_tv_mode_find(conn_state);
 
 	if (!tv_mode)
 		return false;
@@ -1032,7 +982,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	struct intel_tv *intel_tv = enc_to_tv(encoder);
-	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
+	const struct tv_mode *tv_mode = intel_tv_mode_find(conn_state);
 	u32 tv_ctl;
 	u32 scctl1, scctl2, scctl3;
 	int i, j;
@@ -1135,12 +1085,12 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
 	else
 		ysize = 2*tv_mode->nbr_end + 1;
 
-	xpos += intel_tv->margin[TV_MARGIN_LEFT];
-	ypos += intel_tv->margin[TV_MARGIN_TOP];
-	xsize -= (intel_tv->margin[TV_MARGIN_LEFT] +
-		  intel_tv->margin[TV_MARGIN_RIGHT]);
-	ysize -= (intel_tv->margin[TV_MARGIN_TOP] +
-		  intel_tv->margin[TV_MARGIN_BOTTOM]);
+	xpos += conn_state->tv.margins.left;
+	ypos += conn_state->tv.margins.top;
+	xsize -= (conn_state->tv.margins.left +
+		  conn_state->tv.margins.right);
+	ysize -= (conn_state->tv.margins.top +
+		  conn_state->tv.margins.bottom);
 	I915_WRITE(TV_WIN_POS, (xpos<<16)|ypos);
 	I915_WRITE(TV_WIN_SIZE, (xsize<<16)|ysize);
 
@@ -1288,7 +1238,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
 static void intel_tv_find_better_format(struct drm_connector *connector)
 {
 	struct intel_tv *intel_tv = intel_attached_tv(connector);
-	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
+	const struct tv_mode *tv_mode = intel_tv_mode_find(connector->state);
 	int i;
 
 	if ((intel_tv->type == DRM_MODE_CONNECTOR_Component) ==
@@ -1304,9 +1254,7 @@ static void intel_tv_find_better_format(struct drm_connector *connector)
 			break;
 	}
 
-	intel_tv->tv_format = tv_mode->name;
-	drm_object_property_set_value(&connector->base,
-		connector->dev->mode_config.tv_mode_property, i);
+	connector->state->tv.mode = i;
 }
 
 /**
@@ -1347,16 +1295,15 @@ intel_tv_detect(struct drm_connector *connector,
 				connector_status_connected;
 		} else
 			status = connector_status_unknown;
-	} else
-		return connector->status;
 
-	if (status != connector_status_connected)
-		return status;
-
-	intel_tv->type = type;
-	intel_tv_find_better_format(connector);
+		if (status == connector_status_connected) {
+			intel_tv->type = type;
+			intel_tv_find_better_format(connector);
+		}
 
-	return connector_status_connected;
+		return status;
+	} else
+		return connector->status;
 }
 
 static const struct input_res {
@@ -1376,12 +1323,9 @@ static const struct input_res {
  * Chose preferred mode  according to line number of TV format
  */
 static void
-intel_tv_chose_preferred_modes(struct drm_connector *connector,
+intel_tv_choose_preferred_modes(const struct tv_mode *tv_mode,
 			       struct drm_display_mode *mode_ptr)
 {
-	struct intel_tv *intel_tv = intel_attached_tv(connector);
-	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
-
 	if (tv_mode->nbr_end < 480 && mode_ptr->vdisplay == 480)
 		mode_ptr->type |= DRM_MODE_TYPE_PREFERRED;
 	else if (tv_mode->nbr_end > 480) {
@@ -1404,8 +1348,7 @@ static int
 intel_tv_get_modes(struct drm_connector *connector)
 {
 	struct drm_display_mode *mode_ptr;
-	struct intel_tv *intel_tv = intel_attached_tv(connector);
-	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
+	const struct tv_mode *tv_mode = intel_tv_mode_find(connector->state);
 	int j, count = 0;
 	u64 tmp;
 
@@ -1448,7 +1391,7 @@ intel_tv_get_modes(struct drm_connector *connector)
 		mode_ptr->clock = (int) tmp;
 
 		mode_ptr->type = DRM_MODE_TYPE_DRIVER;
-		intel_tv_chose_preferred_modes(connector, mode_ptr);
+		intel_tv_choose_preferred_modes(tv_mode, mode_ptr);
 		drm_mode_probed_add(connector, mode_ptr);
 		count++;
 	}
@@ -1463,74 +1406,47 @@ intel_tv_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-
-static int
-intel_tv_set_property(struct drm_connector *connector, struct drm_property *property,
-		      uint64_t val)
-{
-	struct drm_device *dev = connector->dev;
-	struct intel_tv *intel_tv = intel_attached_tv(connector);
-	struct drm_crtc *crtc = intel_tv->base.base.crtc;
-	int ret = 0;
-	bool changed = false;
-
-	ret = drm_object_property_set_value(&connector->base, property, val);
-	if (ret < 0)
-		goto out;
-
-	if (property == dev->mode_config.tv_left_margin_property &&
-		intel_tv->margin[TV_MARGIN_LEFT] != val) {
-		intel_tv->margin[TV_MARGIN_LEFT] = val;
-		changed = true;
-	} else if (property == dev->mode_config.tv_right_margin_property &&
-		intel_tv->margin[TV_MARGIN_RIGHT] != val) {
-		intel_tv->margin[TV_MARGIN_RIGHT] = val;
-		changed = true;
-	} else if (property == dev->mode_config.tv_top_margin_property &&
-		intel_tv->margin[TV_MARGIN_TOP] != val) {
-		intel_tv->margin[TV_MARGIN_TOP] = val;
-		changed = true;
-	} else if (property == dev->mode_config.tv_bottom_margin_property &&
-		intel_tv->margin[TV_MARGIN_BOTTOM] != val) {
-		intel_tv->margin[TV_MARGIN_BOTTOM] = val;
-		changed = true;
-	} else if (property == dev->mode_config.tv_mode_property) {
-		if (val >= ARRAY_SIZE(tv_modes)) {
-			ret = -EINVAL;
-			goto out;
-		}
-		if (!strcmp(intel_tv->tv_format, tv_modes[val].name))
-			goto out;
-
-		intel_tv->tv_format = tv_modes[val].name;
-		changed = true;
-	} else {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (changed && crtc)
-		intel_crtc_restore_mode(crtc);
-out:
-	return ret;
-}
-
 static const struct drm_connector_funcs intel_tv_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_tv_destroy,
-	.set_property = intel_tv_set_property,
-	.atomic_get_property = intel_connector_atomic_get_property,
+	.set_property = drm_atomic_helper_connector_set_property,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 };
 
+static int intel_tv_atomic_check(struct drm_connector *connector,
+				 struct drm_connector_state *new_state)
+{
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_connector_state *old_state;
+
+	if (!new_state->crtc)
+		return 0;
+
+	old_state = drm_atomic_get_old_connector_state(new_state->state, connector);
+	new_crtc_state = drm_atomic_get_new_crtc_state(new_state->state, new_state->crtc);
+
+	if (old_state->tv.mode != new_state->tv.mode ||
+	    old_state->tv.margins.left != new_state->tv.margins.left ||
+	    old_state->tv.margins.right != new_state->tv.margins.right ||
+	    old_state->tv.margins.top != new_state->tv.margins.top ||
+	    old_state->tv.margins.bottom != new_state->tv.margins.bottom) {
+		/* Force a modeset. */
+
+		new_crtc_state->connectors_changed = true;
+	}
+
+	return 0;
+}
+
 static const struct drm_connector_helper_funcs intel_tv_connector_helper_funcs = {
 	.detect_ctx = intel_tv_detect,
 	.mode_valid = intel_tv_mode_valid,
 	.get_modes = intel_tv_get_modes,
+	.atomic_check = intel_tv_atomic_check,
 };
 
 static const struct drm_encoder_funcs intel_tv_enc_funcs = {
@@ -1548,6 +1464,7 @@ intel_tv_init(struct drm_i915_private *dev_priv)
 	u32 tv_dac_on, tv_dac_off, save_tv_dac;
 	const char *tv_format_names[ARRAY_SIZE(tv_modes)];
 	int i, initial_mode = 0;
+	struct drm_connector_state *state;
 
 	if ((I915_READ(TV_CTL) & TV_FUSE_STATE_MASK) == TV_FUSE_STATE_DISABLED)
 		return;
@@ -1593,6 +1510,7 @@ intel_tv_init(struct drm_i915_private *dev_priv)
 
 	intel_encoder = &intel_tv->base;
 	connector = &intel_connector->base;
+	state = connector->state;
 
 	/* The documentation, for the older chipsets at least, recommend
 	 * using a polling method rather than hotplug detection for TVs.
@@ -1630,12 +1548,12 @@ intel_tv_init(struct drm_i915_private *dev_priv)
 	intel_tv->type = DRM_MODE_CONNECTOR_Unknown;
 
 	/* BIOS margin values */
-	intel_tv->margin[TV_MARGIN_LEFT] = 54;
-	intel_tv->margin[TV_MARGIN_TOP] = 36;
-	intel_tv->margin[TV_MARGIN_RIGHT] = 46;
-	intel_tv->margin[TV_MARGIN_BOTTOM] = 37;
+	state->tv.margins.left = 54;
+	state->tv.margins.top = 36;
+	state->tv.margins.right = 46;
+	state->tv.margins.bottom = 37;
 
-	intel_tv->tv_format = tv_modes[initial_mode].name;
+	state->tv.mode = initial_mode;
 
 	drm_connector_helper_add(connector, &intel_tv_connector_helper_funcs);
 	connector->interlace_allowed = false;
@@ -1649,17 +1567,17 @@ intel_tv_init(struct drm_i915_private *dev_priv)
 				      tv_format_names);
 
 	drm_object_attach_property(&connector->base, dev->mode_config.tv_mode_property,
-				   initial_mode);
+				   state->tv.mode);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.tv_left_margin_property,
-				   intel_tv->margin[TV_MARGIN_LEFT]);
+				   state->tv.margins.left);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.tv_top_margin_property,
-				   intel_tv->margin[TV_MARGIN_TOP]);
+				   state->tv.margins.top);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.tv_right_margin_property,
-				   intel_tv->margin[TV_MARGIN_RIGHT]);
+				   state->tv.margins.right);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.tv_bottom_margin_property,
-				   intel_tv->margin[TV_MARGIN_BOTTOM]);
+				   state->tv.margins.bottom);
 }
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index c117424f1f50..27e072cc96eb 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -94,12 +94,22 @@ void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
 		i915.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
 }
 
+static void guc_write_irq_trigger(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	I915_WRITE(GUC_SEND_INTERRUPT, GUC_SEND_TRIGGER);
+}
+
 void intel_uc_init_early(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
 
+	intel_guc_ct_init_early(&guc->ct);
+
 	mutex_init(&guc->send_mutex);
-	guc->send = intel_guc_send_mmio;
+	guc->send = intel_guc_send_nop;
+	guc->notify = guc_write_irq_trigger;
 }
 
 static void fetch_uc_fw(struct drm_i915_private *dev_priv,
@@ -252,13 +262,81 @@ void intel_uc_fini_fw(struct drm_i915_private *dev_priv)
 	__intel_uc_fw_fini(&dev_priv->huc.fw);
 }
 
+static inline i915_reg_t guc_send_reg(struct intel_guc *guc, u32 i)
+{
+	GEM_BUG_ON(!guc->send_regs.base);
+	GEM_BUG_ON(!guc->send_regs.count);
+	GEM_BUG_ON(i >= guc->send_regs.count);
+
+	return _MMIO(guc->send_regs.base + 4 * i);
+}
+
+static void guc_init_send_regs(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	enum forcewake_domains fw_domains = 0;
+	unsigned int i;
+
+	guc->send_regs.base = i915_mmio_reg_offset(SOFT_SCRATCH(0));
+	guc->send_regs.count = SOFT_SCRATCH_COUNT - 1;
+
+	for (i = 0; i < guc->send_regs.count; i++) {
+		fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+					guc_send_reg(guc, i),
+					FW_REG_READ | FW_REG_WRITE);
+	}
+	guc->send_regs.fw_domains = fw_domains;
+}
+
+static void guc_capture_load_err_log(struct intel_guc *guc)
+{
+	if (!guc->log.vma || i915.guc_log_level < 0)
+		return;
+
+	if (!guc->load_err_log)
+		guc->load_err_log = i915_gem_object_get(guc->log.vma->obj);
+
+	return;
+}
+
+static void guc_free_load_err_log(struct intel_guc *guc)
+{
+	if (guc->load_err_log)
+		i915_gem_object_put(guc->load_err_log);
+}
+
+static int guc_enable_communication(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	guc_init_send_regs(guc);
+
+	if (HAS_GUC_CT(dev_priv))
+		return intel_guc_enable_ct(guc);
+
+	guc->send = intel_guc_send_mmio;
+	return 0;
+}
+
+static void guc_disable_communication(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	if (HAS_GUC_CT(dev_priv))
+		intel_guc_disable_ct(guc);
+
+	guc->send = intel_guc_send_nop;
+}
+
 int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 {
+	struct intel_guc *guc = &dev_priv->guc;
 	int ret, attempts;
 
 	if (!i915.enable_guc_loading)
 		return 0;
 
+	guc_disable_communication(guc);
 	gen9_reset_guc_interrupts(dev_priv);
 
 	/* We need to notify the guc whenever we change the GGTT */
@@ -274,6 +352,11 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 			goto err_guc;
 	}
 
+	/* init WOPCM */
+	I915_WRITE(GUC_WOPCM_SIZE, intel_guc_wopcm_size(dev_priv));
+	I915_WRITE(DMA_GUC_WOPCM_OFFSET,
+		   GUC_WOPCM_OFFSET_VALUE | HUC_LOADING_AGENT_GUC);
+
 	/* WaEnableuKernelHeaderValidFix:skl */
 	/* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */
 	if (IS_GEN9(dev_priv))
@@ -301,7 +384,11 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 
 	/* Did we succeded or run out of retries? */
 	if (ret)
-		goto err_submission;
+		goto err_log_capture;
+
+	ret = guc_enable_communication(guc);
+	if (ret)
+		goto err_log_capture;
 
 	intel_guc_auth_huc(dev_priv);
 	if (i915.enable_guc_submission) {
@@ -325,7 +412,10 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 	 * marks the GPU as wedged until reset).
 	 */
 err_interrupts:
+	guc_disable_communication(guc);
 	gen9_disable_guc_interrupts(dev_priv);
+err_log_capture:
+	guc_capture_load_err_log(guc);
 err_submission:
 	if (i915.enable_guc_submission)
 		i915_guc_submission_fini(dev_priv);
@@ -343,33 +433,36 @@ err_guc:
 		DRM_NOTE("Falling back from GuC submission to execlist mode\n");
 	}
 
+	i915.enable_guc_loading = 0;
+	DRM_NOTE("GuC firmware loading disabled\n");
+
 	return ret;
 }
 
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv)
 {
+	guc_free_load_err_log(&dev_priv->guc);
+
 	if (!i915.enable_guc_loading)
 		return;
 
-	if (i915.enable_guc_submission) {
+	if (i915.enable_guc_submission)
 		i915_guc_submission_disable(dev_priv);
+
+	guc_disable_communication(&dev_priv->guc);
+
+	if (i915.enable_guc_submission) {
 		gen9_disable_guc_interrupts(dev_priv);
 		i915_guc_submission_fini(dev_priv);
 	}
+
 	i915_ggtt_disable_guc(dev_priv);
 }
 
-/*
- * Read GuC command/status register (SOFT_SCRATCH_0)
- * Return true if it contains a response rather than a command
- */
-static bool guc_recv(struct intel_guc *guc, u32 *status)
+int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len)
 {
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-
-	u32 val = I915_READ(SOFT_SCRATCH(0));
-	*status = val;
-	return INTEL_GUC_RECV_IS_RESPONSE(val);
+	WARN(1, "Unexpected send: action=%#x\n", *action);
+	return -ENODEV;
 }
 
 /*
@@ -382,30 +475,33 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len)
 	int i;
 	int ret;
 
-	if (WARN_ON(len < 1 || len > 15))
-		return -EINVAL;
+	GEM_BUG_ON(!len);
+	GEM_BUG_ON(len > guc->send_regs.count);
 
-	mutex_lock(&guc->send_mutex);
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_BLITTER);
+	/* If CT is available, we expect to use MMIO only during init/fini */
+	GEM_BUG_ON(HAS_GUC_CT(dev_priv) &&
+		*action != INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER &&
+		*action != INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER);
 
-	dev_priv->guc.action_count += 1;
-	dev_priv->guc.action_cmd = action[0];
+	mutex_lock(&guc->send_mutex);
+	intel_uncore_forcewake_get(dev_priv, guc->send_regs.fw_domains);
 
 	for (i = 0; i < len; i++)
-		I915_WRITE(SOFT_SCRATCH(i), action[i]);
+		I915_WRITE(guc_send_reg(guc, i), action[i]);
 
-	POSTING_READ(SOFT_SCRATCH(i - 1));
+	POSTING_READ(guc_send_reg(guc, i - 1));
 
-	I915_WRITE(GUC_SEND_INTERRUPT, GUC_SEND_TRIGGER);
+	intel_guc_notify(guc);
 
 	/*
-	 * Fast commands should complete in less than 10us, so sample quickly
-	 * up to that length of time, then switch to a slower sleep-wait loop.
-	 * No inte_guc_send command should ever take longer than 10ms.
+	 * No GuC command should ever take longer than 10ms.
+	 * Fast commands should still complete in 10us.
 	 */
-	ret = wait_for_us(guc_recv(guc, &status), 10);
-	if (ret)
-		ret = wait_for(guc_recv(guc, &status), 10);
+	ret = __intel_wait_for_register_fw(dev_priv,
+					   guc_send_reg(guc, 0),
+					   INTEL_GUC_RECV_MASK,
+					   INTEL_GUC_RECV_MASK,
+					   10, 10, &status);
 	if (status != INTEL_GUC_STATUS_SUCCESS) {
 		/*
 		 * Either the GuC explicitly returned an error (which
@@ -418,13 +514,9 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len)
 		DRM_WARN("INTEL_GUC_SEND: Action 0x%X failed;"
 			 " ret=%d status=0x%08X response=0x%08X\n",
 			 action[0], ret, status, I915_READ(SOFT_SCRATCH(15)));
-
-		dev_priv->guc.action_fail += 1;
-		dev_priv->guc.action_err = ret;
 	}
-	dev_priv->guc.action_status = status;
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_BLITTER);
+	intel_uncore_forcewake_put(dev_priv, guc->send_regs.fw_domains);
 	mutex_unlock(&guc->send_mutex);
 
 	return ret;
diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h
index f84115261ae7..22ae52b17b0f 100644
--- a/drivers/gpu/drm/i915/intel_uc.h
+++ b/drivers/gpu/drm/i915/intel_uc.h
@@ -27,7 +27,7 @@
 #include "intel_guc_fwif.h"
 #include "i915_guc_reg.h"
 #include "intel_ringbuffer.h"
-
+#include "intel_guc_ct.h"
 #include "i915_vma.h"
 
 struct drm_i915_gem_request;
@@ -85,8 +85,6 @@ struct i915_guc_client {
 	uint32_t wq_tail;
 	uint32_t wq_rsvd;
 	uint32_t no_wq_space;
-	uint32_t b_fail;
-	int retcode;
 
 	/* Per-engine counts of GuC submissions */
 	uint64_t submissions[I915_NUM_ENGINES];
@@ -179,6 +177,10 @@ struct intel_guc_log {
 struct intel_guc {
 	struct intel_uc_fw fw;
 	struct intel_guc_log log;
+	struct intel_guc_ct ct;
+
+	/* Log snapshot if GuC errors during load */
+	struct drm_i915_gem_object *load_err_log;
 
 	/* intel_guc_recv interrupt related state */
 	bool interrupts_enabled;
@@ -193,21 +195,21 @@ struct intel_guc {
 	DECLARE_BITMAP(doorbell_bitmap, GUC_NUM_DOORBELLS);
 	uint32_t db_cacheline;		/* Cyclic counter mod pagesize	*/
 
-	/* Action status & statistics */
-	uint64_t action_count;		/* Total commands issued	*/
-	uint32_t action_cmd;		/* Last command word		*/
-	uint32_t action_status;		/* Last return status		*/
-	uint32_t action_fail;		/* Total number of failures	*/
-	int32_t action_err;		/* Last error code		*/
-
-	uint64_t submissions[I915_NUM_ENGINES];
-	uint32_t last_seqno[I915_NUM_ENGINES];
+	/* GuC's FW specific registers used in MMIO send */
+	struct {
+		u32 base;
+		unsigned int count;
+		enum forcewake_domains fw_domains;
+	} send_regs;
 
 	/* To serialize the intel_guc_send actions */
 	struct mutex send_mutex;
 
 	/* GuC's FW specific send function */
 	int (*send)(struct intel_guc *guc, const u32 *data, u32 len);
+
+	/* GuC's FW specific notify function */
+	void (*notify)(struct intel_guc *guc);
 };
 
 struct intel_huc {
@@ -225,12 +227,19 @@ void intel_uc_fini_fw(struct drm_i915_private *dev_priv);
 int intel_uc_init_hw(struct drm_i915_private *dev_priv);
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv);
 int intel_guc_sample_forcewake(struct intel_guc *guc);
+int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len);
 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len);
+
 static inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)
 {
 	return guc->send(guc, action, len);
 }
 
+static inline void intel_guc_notify(struct intel_guc *guc)
+{
+	guc->notify(guc);
+}
+
 /* intel_guc_loader.c */
 int intel_guc_select_fw(struct intel_guc *guc);
 int intel_guc_init_hw(struct intel_guc *guc);
@@ -264,7 +273,7 @@ static inline u32 guc_ggtt_offset(struct i915_vma *vma)
 
 /* intel_huc.c */
 void intel_huc_select_fw(struct intel_huc *huc);
-int intel_huc_init_hw(struct intel_huc *huc);
+void intel_huc_init_hw(struct intel_huc *huc);
 void intel_guc_auth_huc(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 6d1ea26b2493..9882724bc2b6 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -29,6 +29,7 @@
 #include <linux/pm_runtime.h>
 
 #define FORCEWAKE_ACK_TIMEOUT_MS 50
+#define GT_FIFO_TIMEOUT_MS	 10
 
 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
 
@@ -172,22 +173,6 @@ static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
 	__gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
-static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
-{
-	u32 gtfifodbg;
-
-	gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
-	if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
-		__raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
-}
-
-static void fw_domains_put_with_fifo(struct drm_i915_private *dev_priv,
-				     enum forcewake_domains fw_domains)
-{
-	fw_domains_put(dev_priv, fw_domains);
-	gen6_gt_check_fifodbg(dev_priv);
-}
-
 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
 {
 	u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
@@ -195,30 +180,27 @@ static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
 	return count & GT_FIFO_FREE_ENTRIES_MASK;
 }
 
-static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
+static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 {
-	int ret = 0;
+	u32 n;
 
 	/* On VLV, FIFO will be shared by both SW and HW.
 	 * So, we need to read the FREE_ENTRIES everytime */
 	if (IS_VALLEYVIEW(dev_priv))
-		dev_priv->uncore.fifo_count = fifo_free_entries(dev_priv);
-
-	if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
-		int loop = 500;
-		u32 fifo = fifo_free_entries(dev_priv);
-
-		while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
-			udelay(10);
-			fifo = fifo_free_entries(dev_priv);
+		n = fifo_free_entries(dev_priv);
+	else
+		n = dev_priv->uncore.fifo_count;
+
+	if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
+		if (wait_for_atomic((n = fifo_free_entries(dev_priv)) >
+				    GT_FIFO_NUM_RESERVED_ENTRIES,
+				    GT_FIFO_TIMEOUT_MS)) {
+			DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n);
+			return;
 		}
-		if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
-			++ret;
-		dev_priv->uncore.fifo_count = fifo;
 	}
-	dev_priv->uncore.fifo_count--;
 
-	return ret;
+	dev_priv->uncore.fifo_count = n - 1;
 }
 
 static enum hrtimer_restart
@@ -232,6 +214,9 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
 
 	assert_rpm_device_not_suspended(dev_priv);
 
+	if (xchg(&domain->active, false))
+		return HRTIMER_RESTART;
+
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 	if (WARN_ON(domain->wake_count == 0))
 		domain->wake_count++;
@@ -262,6 +247,7 @@ static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 		active_domains = 0;
 
 		for_each_fw_domain(domain, dev_priv, tmp) {
+			smp_store_mb(domain->active, false);
 			if (hrtimer_cancel(&domain->timer) == 0)
 				continue;
 
@@ -384,31 +370,44 @@ vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 }
 
 static bool
+gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv)
+{
+	u32 fifodbg;
+
+	fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
+
+	if (unlikely(fifodbg)) {
+		DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg);
+		__raw_i915_write32(dev_priv, GTFIFODBG, fifodbg);
+	}
+
+	return fifodbg;
+}
+
+static bool
 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 {
+	bool ret = false;
+
 	if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
-		return fpga_check_for_unclaimed_mmio(dev_priv);
+		ret |= fpga_check_for_unclaimed_mmio(dev_priv);
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		return vlv_check_for_unclaimed_mmio(dev_priv);
+		ret |= vlv_check_for_unclaimed_mmio(dev_priv);
 
-	return false;
+	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
+		ret |= gen6_check_for_fifo_debug(dev_priv);
+
+	return ret;
 }
 
 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 					  bool restore_forcewake)
 {
-	struct intel_device_info *info = mkwrite_device_info(dev_priv);
-
 	/* clear out unclaimed reg detection bit */
 	if (check_for_unclaimed_mmio(dev_priv))
 		DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
 
-	/* clear out old GT FIFO errors */
-	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
-		__raw_i915_write32(dev_priv, GTFIFODBG,
-				   __raw_i915_read32(dev_priv, GTFIFODBG));
-
 	/* WaDisableShadowRegForCpd:chv */
 	if (IS_CHERRYVIEW(dev_priv)) {
 		__raw_i915_write32(dev_priv, GTFIFOCTL,
@@ -417,9 +416,6 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 				   GT_FIFO_CTL_RC6_POLICY_STALL);
 	}
 
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST))
-		info->has_decoupled_mmio = false;
-
 	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
 }
 
@@ -454,9 +450,12 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 
 	fw_domains &= dev_priv->uncore.fw_domains;
 
-	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp)
-		if (domain->wake_count++)
+	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
+		if (domain->wake_count++) {
 			fw_domains &= ~domain->mask;
+			domain->active = true;
+		}
+	}
 
 	if (fw_domains)
 		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
@@ -521,8 +520,10 @@ static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 		if (WARN_ON(domain->wake_count == 0))
 			continue;
 
-		if (--domain->wake_count)
+		if (--domain->wake_count) {
+			domain->active = true;
 			continue;
+		}
 
 		fw_domain_arm_timer(domain);
 	}
@@ -804,66 +805,6 @@ unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 	__unclaimed_reg_debug(dev_priv, reg, read, before);
 }
 
-static const enum decoupled_power_domain fw2dpd_domain[] = {
-	GEN9_DECOUPLED_PD_RENDER,
-	GEN9_DECOUPLED_PD_BLITTER,
-	GEN9_DECOUPLED_PD_ALL,
-	GEN9_DECOUPLED_PD_MEDIA,
-	GEN9_DECOUPLED_PD_ALL,
-	GEN9_DECOUPLED_PD_ALL,
-	GEN9_DECOUPLED_PD_ALL
-};
-
-/*
- * Decoupled MMIO access for only 1 DWORD
- */
-static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv,
-					 u32 reg,
-					 enum forcewake_domains fw_domain,
-					 enum decoupled_ops operation)
-{
-	enum decoupled_power_domain dp_domain;
-	u32 ctrl_reg_data = 0;
-
-	dp_domain = fw2dpd_domain[fw_domain - 1];
-
-	ctrl_reg_data |= reg;
-	ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT);
-	ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT);
-	ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO;
-	__raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data);
-
-	if (wait_for_atomic((__raw_i915_read32(dev_priv,
-			    GEN9_DECOUPLED_REG0_DW1) &
-			    GEN9_DECOUPLED_DW1_GO) == 0,
-			    FORCEWAKE_ACK_TIMEOUT_MS))
-		DRM_ERROR("Decoupled MMIO wait timed out\n");
-}
-
-static inline u32
-__gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv,
-			     u32 reg,
-			     enum forcewake_domains fw_domain)
-{
-	__gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
-				     GEN9_DECOUPLED_OP_READ);
-
-	return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0);
-}
-
-static inline void
-__gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv,
-			    u32 reg, u32 data,
-			    enum forcewake_domains fw_domain)
-{
-
-	__raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data);
-
-	__gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
-				     GEN9_DECOUPLED_OP_WRITE);
-}
-
-
 #define GEN2_READ_HEADER(x) \
 	u##x val = 0; \
 	assert_rpm_wakelock_held(dev_priv);
@@ -960,28 +901,6 @@ func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) {
 #define __gen6_read(x) __gen_read(gen6, x)
 #define __fwtable_read(x) __gen_read(fwtable, x)
 
-#define __gen9_decoupled_read(x) \
-static u##x \
-gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \
-		       i915_reg_t reg, bool trace) { \
-	enum forcewake_domains fw_engine; \
-	GEN6_READ_HEADER(x); \
-	fw_engine = __fwtable_reg_read_fw_domains(offset); \
-	if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \
-		unsigned i; \
-		u32 *ptr_data = (u32 *) &val; \
-		for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \
-			*ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \
-								 offset, \
-								 fw_engine); \
-	} else { \
-		val = __raw_i915_read##x(dev_priv, reg); \
-	} \
-	GEN6_READ_FOOTER; \
-}
-
-__gen9_decoupled_read(32)
-__gen9_decoupled_read(64)
 __fwtable_read(8)
 __fwtable_read(16)
 __fwtable_read(32)
@@ -1047,15 +966,10 @@ __gen2_write(32)
 #define __gen6_write(x) \
 static void \
 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
-	u32 __fifo_ret = 0; \
 	GEN6_WRITE_HEADER; \
-	if (NEEDS_FORCE_WAKE(offset)) { \
-		__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
-	} \
+	if (NEEDS_FORCE_WAKE(offset)) \
+		__gen6_gt_wait_for_fifo(dev_priv); \
 	__raw_i915_write##x(dev_priv, reg, val); \
-	if (unlikely(__fifo_ret)) { \
-		gen6_gt_check_fifodbg(dev_priv); \
-	} \
 	GEN6_WRITE_FOOTER; \
 }
 
@@ -1073,25 +987,6 @@ func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, boo
 #define __gen8_write(x) __gen_write(gen8, x)
 #define __fwtable_write(x) __gen_write(fwtable, x)
 
-#define __gen9_decoupled_write(x) \
-static void \
-gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \
-			i915_reg_t reg, u##x val, \
-		bool trace) { \
-	enum forcewake_domains fw_engine; \
-	GEN6_WRITE_HEADER; \
-	fw_engine = __fwtable_reg_write_fw_domains(offset); \
-	if (fw_engine & ~dev_priv->uncore.fw_domains_active) \
-		__gen9_decoupled_mmio_write(dev_priv, \
-					    offset, \
-					    val, \
-					    fw_engine); \
-	else \
-		__raw_i915_write##x(dev_priv, reg, val); \
-	GEN6_WRITE_FOOTER; \
-}
-
-__gen9_decoupled_write(32)
 __fwtable_write(8)
 __fwtable_write(16)
 __fwtable_write(32)
@@ -1108,19 +1003,19 @@ __gen6_write(32)
 #undef GEN6_WRITE_FOOTER
 #undef GEN6_WRITE_HEADER
 
-#define ASSIGN_WRITE_MMIO_VFUNCS(x) \
+#define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
 do { \
-	dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
-	dev_priv->uncore.funcs.mmio_writew = x##_write16; \
-	dev_priv->uncore.funcs.mmio_writel = x##_write32; \
+	(i915)->uncore.funcs.mmio_writeb = x##_write8; \
+	(i915)->uncore.funcs.mmio_writew = x##_write16; \
+	(i915)->uncore.funcs.mmio_writel = x##_write32; \
 } while (0)
 
-#define ASSIGN_READ_MMIO_VFUNCS(x) \
+#define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
 do { \
-	dev_priv->uncore.funcs.mmio_readb = x##_read8; \
-	dev_priv->uncore.funcs.mmio_readw = x##_read16; \
-	dev_priv->uncore.funcs.mmio_readl = x##_read32; \
-	dev_priv->uncore.funcs.mmio_readq = x##_read64; \
+	(i915)->uncore.funcs.mmio_readb = x##_read8; \
+	(i915)->uncore.funcs.mmio_readw = x##_read16; \
+	(i915)->uncore.funcs.mmio_readl = x##_read32; \
+	(i915)->uncore.funcs.mmio_readq = x##_read64; \
 } while (0)
 
 
@@ -1190,11 +1085,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 			       FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
-		if (!IS_CHERRYVIEW(dev_priv))
-			dev_priv->uncore.funcs.force_wake_put =
-				fw_domains_put_with_fifo;
-		else
-			dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
+		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
@@ -1202,11 +1093,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		dev_priv->uncore.funcs.force_wake_get =
 			fw_domains_get_with_thread_status;
-		if (IS_HASWELL(dev_priv))
-			dev_priv->uncore.funcs.force_wake_put =
-				fw_domains_put_with_fifo;
-		else
-			dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
+		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
 	} else if (IS_IVYBRIDGE(dev_priv)) {
@@ -1223,8 +1110,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 		 */
 		dev_priv->uncore.funcs.force_wake_get =
 			fw_domains_get_with_thread_status;
-		dev_priv->uncore.funcs.force_wake_put =
-			fw_domains_put_with_fifo;
+		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 
 		/* We need to init first for ECOBUS access and then
 		 * determine later if we want to reinit, in case of MT access is
@@ -1242,7 +1128,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 		spin_lock_irq(&dev_priv->uncore.lock);
 		fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER);
 		ecobus = __raw_i915_read32(dev_priv, ECOBUS);
-		fw_domains_put_with_fifo(dev_priv, FORCEWAKE_RENDER);
+		fw_domains_put(dev_priv, FORCEWAKE_RENDER);
 		spin_unlock_irq(&dev_priv->uncore.lock);
 
 		if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
@@ -1254,8 +1140,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 	} else if (IS_GEN6(dev_priv)) {
 		dev_priv->uncore.funcs.force_wake_get =
 			fw_domains_get_with_thread_status;
-		dev_priv->uncore.funcs.force_wake_put =
-			fw_domains_put_with_fifo;
+		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE, FORCEWAKE_ACK);
 	}
@@ -1310,42 +1195,34 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 		i915_pmic_bus_access_notifier;
 
 	if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) {
-		ASSIGN_WRITE_MMIO_VFUNCS(gen2);
-		ASSIGN_READ_MMIO_VFUNCS(gen2);
+		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2);
+		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2);
 	} else if (IS_GEN5(dev_priv)) {
-		ASSIGN_WRITE_MMIO_VFUNCS(gen5);
-		ASSIGN_READ_MMIO_VFUNCS(gen5);
+		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5);
+		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5);
 	} else if (IS_GEN(dev_priv, 6, 7)) {
-		ASSIGN_WRITE_MMIO_VFUNCS(gen6);
+		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6);
 
 		if (IS_VALLEYVIEW(dev_priv)) {
 			ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
-			ASSIGN_READ_MMIO_VFUNCS(fwtable);
+			ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
 		} else {
-			ASSIGN_READ_MMIO_VFUNCS(gen6);
+			ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
 		}
 	} else if (IS_GEN8(dev_priv)) {
 		if (IS_CHERRYVIEW(dev_priv)) {
 			ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
-			ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
-			ASSIGN_READ_MMIO_VFUNCS(fwtable);
+			ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
+			ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
 
 		} else {
-			ASSIGN_WRITE_MMIO_VFUNCS(gen8);
-			ASSIGN_READ_MMIO_VFUNCS(gen6);
+			ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
+			ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
 		}
 	} else {
 		ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
-		ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
-		ASSIGN_READ_MMIO_VFUNCS(fwtable);
-		if (HAS_DECOUPLED_MMIO(dev_priv)) {
-			dev_priv->uncore.funcs.mmio_readl =
-						gen9_decoupled_read32;
-			dev_priv->uncore.funcs.mmio_readq =
-						gen9_decoupled_read64;
-			dev_priv->uncore.funcs.mmio_writel =
-						gen9_decoupled_write32;
-		}
+		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
+		ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
 	}
 
 	iosf_mbi_register_pmic_bus_access_notifier(
@@ -1353,8 +1230,6 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 
 	i915_check_and_clear_faults(dev_priv);
 }
-#undef ASSIGN_WRITE_MMIO_VFUNCS
-#undef ASSIGN_READ_MMIO_VFUNCS
 
 void intel_uncore_fini(struct drm_i915_private *dev_priv)
 {
@@ -1435,9 +1310,39 @@ out:
 	return ret;
 }
 
-static int i915_reset_complete(struct pci_dev *pdev)
+static void gen3_stop_rings(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, dev_priv, id) {
+		const u32 base = engine->mmio_base;
+		const i915_reg_t mode = RING_MI_MODE(base);
+
+		I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
+		if (intel_wait_for_register_fw(dev_priv,
+					       mode,
+					       MODE_IDLE,
+					       MODE_IDLE,
+					       500))
+			DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
+					 engine->name);
+
+		I915_WRITE_FW(RING_CTL(base), 0);
+		I915_WRITE_FW(RING_HEAD(base), 0);
+		I915_WRITE_FW(RING_TAIL(base), 0);
+
+		/* Check acts as a post */
+		if (I915_READ_FW(RING_HEAD(base)) != 0)
+			DRM_DEBUG_DRIVER("%s: ring head not parked\n",
+					 engine->name);
+	}
+}
+
+static bool i915_reset_complete(struct pci_dev *pdev)
 {
 	u8 gdrst;
+
 	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
 	return (gdrst & GRDOM_RESET_STATUS) == 0;
 }
@@ -1448,15 +1353,16 @@ static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask
 
 	/* assert reset for at least 20 usec */
 	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
-	udelay(20);
+	usleep_range(50, 200);
 	pci_write_config_byte(pdev, I915_GDRST, 0);
 
 	return wait_for(i915_reset_complete(pdev), 500);
 }
 
-static int g4x_reset_complete(struct pci_dev *pdev)
+static bool g4x_reset_complete(struct pci_dev *pdev)
 {
 	u8 gdrst;
+
 	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
 	return (gdrst & GRDOM_RESET_ENABLE) == 0;
 }
@@ -1464,6 +1370,10 @@ static int g4x_reset_complete(struct pci_dev *pdev)
 static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 {
 	struct pci_dev *pdev = dev_priv->drm.pdev;
+
+	/* Stop engines before we reset; see g4x_do_reset() below for why. */
+	gen3_stop_rings(dev_priv);
+
 	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
 	return wait_for(g4x_reset_complete(pdev), 500);
 }
@@ -1473,29 +1383,41 @@ static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	int ret;
 
-	pci_write_config_byte(pdev, I915_GDRST,
-			      GRDOM_RENDER | GRDOM_RESET_ENABLE);
-	ret =  wait_for(g4x_reset_complete(pdev), 500);
-	if (ret)
-		return ret;
-
 	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
-	I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
+	I915_WRITE(VDECCLK_GATE_D,
+		   I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
 	POSTING_READ(VDECCLK_GATE_D);
 
+	/* We stop engines, otherwise we might get failed reset and a
+	 * dead gpu (on elk).
+	 * WaMediaResetMainRingCleanup:ctg,elk (presumably)
+	 */
+	gen3_stop_rings(dev_priv);
+
 	pci_write_config_byte(pdev, I915_GDRST,
 			      GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 	ret =  wait_for(g4x_reset_complete(pdev), 500);
-	if (ret)
-		return ret;
+	if (ret) {
+		DRM_DEBUG_DRIVER("Wait for media reset failed\n");
+		goto out;
+	}
 
-	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
-	I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
-	POSTING_READ(VDECCLK_GATE_D);
+	pci_write_config_byte(pdev, I915_GDRST,
+			      GRDOM_RENDER | GRDOM_RESET_ENABLE);
+	ret =  wait_for(g4x_reset_complete(pdev), 500);
+	if (ret) {
+		DRM_DEBUG_DRIVER("Wait for render reset failed\n");
+		goto out;
+	}
 
+out:
 	pci_write_config_byte(pdev, I915_GDRST, 0);
 
-	return 0;
+	I915_WRITE(VDECCLK_GATE_D,
+		   I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
+	POSTING_READ(VDECCLK_GATE_D);
+
+	return ret;
 }
 
 static int ironlake_do_reset(struct drm_i915_private *dev_priv,
@@ -1503,41 +1425,51 @@ static int ironlake_do_reset(struct drm_i915_private *dev_priv,
 {
 	int ret;
 
-	I915_WRITE(ILK_GDSR,
-		   ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
+	I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
 	ret = intel_wait_for_register(dev_priv,
 				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
 				      500);
-	if (ret)
-		return ret;
+	if (ret) {
+		DRM_DEBUG_DRIVER("Wait for render reset failed\n");
+		goto out;
+	}
 
-	I915_WRITE(ILK_GDSR,
-		   ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
+	I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
 	ret = intel_wait_for_register(dev_priv,
 				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
 				      500);
-	if (ret)
-		return ret;
+	if (ret) {
+		DRM_DEBUG_DRIVER("Wait for media reset failed\n");
+		goto out;
+	}
 
+out:
 	I915_WRITE(ILK_GDSR, 0);
-
-	return 0;
+	POSTING_READ(ILK_GDSR);
+	return ret;
 }
 
 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
 				u32 hw_domain_mask)
 {
+	int err;
+
 	/* GEN6_GDRST is not in the gt power well, no need to check
 	 * for fifo space for the write or forcewake the chip for
 	 * the read
 	 */
 	__raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
 
-	/* Spin waiting for the device to ack the reset requests */
-	return intel_wait_for_register_fw(dev_priv,
+	/* Wait for the device to ack the reset requests */
+	err = intel_wait_for_register_fw(dev_priv,
 					  GEN6_GDRST, hw_domain_mask, 0,
 					  500);
+	if (err)
+		DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
+				 hw_domain_mask);
+
+	return err;
 }
 
 /**
@@ -1585,19 +1517,23 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
 }
 
 /**
- * intel_wait_for_register_fw - wait until register matches expected state
+ * __intel_wait_for_register_fw - wait until register matches expected state
  * @dev_priv: the i915 device
  * @reg: the register to read
  * @mask: mask to apply to register value
  * @value: expected value
- * @timeout_ms: timeout in millisecond
+ * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
+ * @slow_timeout_ms: slow timeout in millisecond
+ * @out_value: optional placeholder to hold registry value
  *
  * This routine waits until the target register @reg contains the expected
  * @value after applying the @mask, i.e. it waits until ::
  *
  *     (I915_READ_FW(reg) & mask) == value
  *
- * Otherwise, the wait will timeout after @timeout_ms milliseconds.
+ * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
+ * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
+ * must be not larger than 20,0000 microseconds.
  *
  * Note that this routine assumes the caller holds forcewake asserted, it is
  * not suitable for very long waits. See intel_wait_for_register() if you
@@ -1606,16 +1542,31 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
  *
  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
  */
-int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
-			       i915_reg_t reg,
-			       const u32 mask,
-			       const u32 value,
-			       const unsigned long timeout_ms)
-{
-#define done ((I915_READ_FW(reg) & mask) == value)
-	int ret = wait_for_us(done, 2);
-	if (ret)
-		ret = wait_for(done, timeout_ms);
+int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+				 i915_reg_t reg,
+				 u32 mask,
+				 u32 value,
+				 unsigned int fast_timeout_us,
+				 unsigned int slow_timeout_ms,
+				 u32 *out_value)
+{
+	u32 uninitialized_var(reg_value);
+#define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
+	int ret;
+
+	/* Catch any overuse of this function */
+	might_sleep_if(slow_timeout_ms);
+	GEM_BUG_ON(fast_timeout_us > 20000);
+
+	ret = -ETIMEDOUT;
+	if (fast_timeout_us && fast_timeout_us <= 20000)
+		ret = _wait_for_atomic(done, fast_timeout_us, 0);
+	if (ret && slow_timeout_ms)
+		ret = wait_for(done, slow_timeout_ms);
+
+	if (out_value)
+		*out_value = reg_value;
+
 	return ret;
 #undef done
 }
@@ -1639,18 +1590,26 @@ int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
  */
 int intel_wait_for_register(struct drm_i915_private *dev_priv,
 			    i915_reg_t reg,
-			    const u32 mask,
-			    const u32 value,
-			    const unsigned long timeout_ms)
+			    u32 mask,
+			    u32 value,
+			    unsigned int timeout_ms)
 {
-
 	unsigned fw =
 		intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
 	int ret;
 
-	intel_uncore_forcewake_get(dev_priv, fw);
-	ret = wait_for_us((I915_READ_FW(reg) & mask) == value, 2);
-	intel_uncore_forcewake_put(dev_priv, fw);
+	might_sleep();
+
+	spin_lock_irq(&dev_priv->uncore.lock);
+	intel_uncore_forcewake_get__locked(dev_priv, fw);
+
+	ret = __intel_wait_for_register_fw(dev_priv,
+					   reg, mask, value,
+					   2, 0, NULL);
+
+	intel_uncore_forcewake_put__locked(dev_priv, fw);
+	spin_unlock_irq(&dev_priv->uncore.lock);
+
 	if (ret)
 		ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
 			       timeout_ms);
@@ -1658,7 +1617,7 @@ int intel_wait_for_register(struct drm_i915_private *dev_priv,
 	return ret;
 }
 
-static int gen8_request_engine_reset(struct intel_engine_cs *engine)
+static int gen8_reset_engine_start(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 	int ret;
@@ -1677,7 +1636,7 @@ static int gen8_request_engine_reset(struct intel_engine_cs *engine)
 	return ret;
 }
 
-static void gen8_unrequest_engine_reset(struct intel_engine_cs *engine)
+static void gen8_reset_engine_cancel(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
@@ -1692,14 +1651,14 @@ static int gen8_reset_engines(struct drm_i915_private *dev_priv,
 	unsigned int tmp;
 
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
-		if (gen8_request_engine_reset(engine))
+		if (gen8_reset_engine_start(engine))
 			goto not_ready;
 
 	return gen6_reset_engines(dev_priv, engine_mask);
 
 not_ready:
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
-		gen8_unrequest_engine_reset(engine);
+		gen8_reset_engine_cancel(engine);
 
 	return -EIO;
 }
@@ -1730,8 +1689,11 @@ static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
 int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 {
 	reset_func reset;
+	int retry;
 	int ret;
 
+	might_sleep();
+
 	reset = intel_get_gpu_reset(dev_priv);
 	if (reset == NULL)
 		return -ENODEV;
@@ -1740,7 +1702,13 @@ int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 	 * request may be dropped and never completes (causing -EIO).
 	 */
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
-	ret = reset(dev_priv, engine_mask);
+	for (retry = 0; retry < 3; retry++) {
+		ret = reset(dev_priv, engine_mask);
+		if (ret != -ETIMEDOUT)
+			break;
+
+		cond_resched();
+	}
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
 	return ret;
@@ -1754,17 +1722,12 @@ bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
 int intel_guc_reset(struct drm_i915_private *dev_priv)
 {
 	int ret;
-	unsigned long irqflags;
 
 	if (!HAS_GUC(dev_priv))
 		return -EINVAL;
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
-
 	ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);
-
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
 	return ret;
@@ -1873,5 +1836,6 @@ intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/mock_uncore.c"
 #include "selftests/intel_uncore.c"
 #endif
diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
new file mode 100644
index 000000000000..5f90278da461
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_uncore.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __INTEL_UNCORE_H__
+#define __INTEL_UNCORE_H__
+
+struct drm_i915_private;
+
+enum forcewake_domain_id {
+	FW_DOMAIN_ID_RENDER = 0,
+	FW_DOMAIN_ID_BLITTER,
+	FW_DOMAIN_ID_MEDIA,
+
+	FW_DOMAIN_ID_COUNT
+};
+
+enum forcewake_domains {
+	FORCEWAKE_RENDER = BIT(FW_DOMAIN_ID_RENDER),
+	FORCEWAKE_BLITTER = BIT(FW_DOMAIN_ID_BLITTER),
+	FORCEWAKE_MEDIA	= BIT(FW_DOMAIN_ID_MEDIA),
+	FORCEWAKE_ALL = (FORCEWAKE_RENDER |
+			 FORCEWAKE_BLITTER |
+			 FORCEWAKE_MEDIA)
+};
+
+struct intel_uncore_funcs {
+	void (*force_wake_get)(struct drm_i915_private *dev_priv,
+			       enum forcewake_domains domains);
+	void (*force_wake_put)(struct drm_i915_private *dev_priv,
+			       enum forcewake_domains domains);
+
+	uint8_t  (*mmio_readb)(struct drm_i915_private *dev_priv,
+			       i915_reg_t r, bool trace);
+	uint16_t (*mmio_readw)(struct drm_i915_private *dev_priv,
+			       i915_reg_t r, bool trace);
+	uint32_t (*mmio_readl)(struct drm_i915_private *dev_priv,
+			       i915_reg_t r, bool trace);
+	uint64_t (*mmio_readq)(struct drm_i915_private *dev_priv,
+			       i915_reg_t r, bool trace);
+
+	void (*mmio_writeb)(struct drm_i915_private *dev_priv,
+			    i915_reg_t r, uint8_t val, bool trace);
+	void (*mmio_writew)(struct drm_i915_private *dev_priv,
+			    i915_reg_t r, uint16_t val, bool trace);
+	void (*mmio_writel)(struct drm_i915_private *dev_priv,
+			    i915_reg_t r, uint32_t val, bool trace);
+};
+
+struct intel_forcewake_range {
+	u32 start;
+	u32 end;
+
+	enum forcewake_domains domains;
+};
+
+struct intel_uncore {
+	spinlock_t lock; /** lock is also taken in irq contexts. */
+
+	const struct intel_forcewake_range *fw_domains_table;
+	unsigned int fw_domains_table_entries;
+
+	struct notifier_block pmic_bus_access_nb;
+	struct intel_uncore_funcs funcs;
+
+	unsigned int fifo_count;
+
+	enum forcewake_domains fw_domains;
+	enum forcewake_domains fw_domains_active;
+
+	u32 fw_set;
+	u32 fw_clear;
+	u32 fw_reset;
+
+	struct intel_uncore_forcewake_domain {
+		enum forcewake_domain_id id;
+		enum forcewake_domains mask;
+		unsigned int wake_count;
+		bool active;
+		struct hrtimer timer;
+		i915_reg_t reg_set;
+		i915_reg_t reg_ack;
+	} fw_domain[FW_DOMAIN_ID_COUNT];
+
+	int unclaimed_mmio_check;
+};
+
+/* Iterate over initialised fw domains */
+#define for_each_fw_domain_masked(domain__, mask__, dev_priv__, tmp__) \
+	for (tmp__ = (mask__); \
+	     tmp__ ? (domain__ = &(dev_priv__)->uncore.fw_domain[__mask_next_bit(tmp__)]), 1 : 0;)
+
+#define for_each_fw_domain(domain__, dev_priv__, tmp__) \
+	for_each_fw_domain_masked(domain__, (dev_priv__)->uncore.fw_domains, dev_priv__, tmp__)
+
+
+void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
+void intel_uncore_init(struct drm_i915_private *dev_priv);
+bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
+bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
+void intel_uncore_fini(struct drm_i915_private *dev_priv);
+void intel_uncore_suspend(struct drm_i915_private *dev_priv);
+void intel_uncore_resume_early(struct drm_i915_private *dev_priv);
+
+u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv);
+void assert_forcewakes_inactive(struct drm_i915_private *dev_priv);
+const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id);
+
+enum forcewake_domains
+intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
+			       i915_reg_t reg, unsigned int op);
+#define FW_REG_READ  (1)
+#define FW_REG_WRITE (2)
+
+void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
+				enum forcewake_domains domains);
+void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
+				enum forcewake_domains domains);
+/* Like above but the caller must manage the uncore.lock itself.
+ * Must be used with I915_READ_FW and friends.
+ */
+void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
+					enum forcewake_domains domains);
+void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
+					enum forcewake_domains domains);
+
+int intel_wait_for_register(struct drm_i915_private *dev_priv,
+			    i915_reg_t reg,
+			    u32 mask,
+			    u32 value,
+			    unsigned int timeout_ms);
+int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+				 i915_reg_t reg,
+				 u32 mask,
+				 u32 value,
+				 unsigned int fast_timeout_us,
+				 unsigned int slow_timeout_ms,
+				 u32 *out_value);
+static inline
+int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+			       i915_reg_t reg,
+			       u32 mask,
+			       u32 value,
+			       unsigned int timeout_ms)
+{
+	return __intel_wait_for_register_fw(dev_priv, reg, mask, value,
+					    2, timeout_ms, NULL);
+}
+
+#endif /* !__INTEL_UNCORE_H__ */
diff --git a/drivers/gpu/drm/i915/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
index 4e681fc13be4..caf76af36aba 100644
--- a/drivers/gpu/drm/i915/selftests/huge_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
@@ -126,9 +126,11 @@ huge_gem_object(struct drm_i915_private *i915,
 	drm_gem_private_object_init(&i915->drm, &obj->base, dma_size);
 	i915_gem_object_init(obj, &huge_ops);
 
-	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	obj->cache_coherent = i915_gem_object_is_coherent(obj);
+	obj->cache_dirty = !obj->cache_coherent;
 	obj->scratch = phys_size;
 
 	return obj;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index f08d0179b3df..95d4aebc0181 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -138,10 +138,7 @@ static int wc_set(struct drm_i915_gem_object *obj,
 	typeof(v) *map;
 	int err;
 
-	/* XXX GTT write followed by WC write go missing */
-	i915_gem_object_flush_gtt_write_domain(obj);
-
-	err = i915_gem_object_set_to_gtt_domain(obj, true);
+	err = i915_gem_object_set_to_wc_domain(obj, true);
 	if (err)
 		return err;
 
@@ -162,10 +159,7 @@ static int wc_get(struct drm_i915_gem_object *obj,
 	typeof(v) map;
 	int err;
 
-	/* XXX WC write followed by GTT write go missing */
-	i915_gem_object_flush_gtt_write_domain(obj);
-
-	err = i915_gem_object_set_to_gtt_domain(obj, false);
+	err = i915_gem_object_set_to_wc_domain(obj, false);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
index 817bef74bbcb..d15cc9d3a5cd 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
@@ -271,6 +271,105 @@ err_obj:
 	return err;
 }
 
+static int igt_dmabuf_export_kmap(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct drm_i915_gem_object *obj;
+	struct dma_buf *dmabuf;
+	void *ptr;
+	int err;
+
+	obj = i915_gem_object_create(i915, 2*PAGE_SIZE);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0);
+	i915_gem_object_put(obj);
+	if (IS_ERR(dmabuf)) {
+		err = PTR_ERR(dmabuf);
+		pr_err("i915_gem_prime_export failed with err=%d\n", err);
+		return err;
+	}
+
+	ptr = dma_buf_kmap(dmabuf, 0);
+	if (!ptr) {
+		pr_err("dma_buf_kmap failed\n");
+		err = -ENOMEM;
+		goto err;
+	}
+
+	if (memchr_inv(ptr, 0, PAGE_SIZE)) {
+		dma_buf_kunmap(dmabuf, 0, ptr);
+		pr_err("Exported page[0] not initialiased to zero!\n");
+		err = -EINVAL;
+		goto err;
+	}
+
+	memset(ptr, 0xc5, PAGE_SIZE);
+	dma_buf_kunmap(dmabuf, 0, ptr);
+
+	ptr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(ptr)) {
+		err = PTR_ERR(ptr);
+		pr_err("i915_gem_object_pin_map failed with err=%d\n", err);
+		goto err;
+	}
+	memset(ptr + PAGE_SIZE, 0xaa, PAGE_SIZE);
+	i915_gem_object_unpin_map(obj);
+
+	ptr = dma_buf_kmap(dmabuf, 1);
+	if (!ptr) {
+		pr_err("dma_buf_kmap failed\n");
+		err = -ENOMEM;
+		goto err;
+	}
+
+	if (memchr_inv(ptr, 0xaa, PAGE_SIZE)) {
+		dma_buf_kunmap(dmabuf, 1, ptr);
+		pr_err("Exported page[1] not set to 0xaa!\n");
+		err = -EINVAL;
+		goto err;
+	}
+
+	memset(ptr, 0xc5, PAGE_SIZE);
+	dma_buf_kunmap(dmabuf, 1, ptr);
+
+	ptr = dma_buf_kmap(dmabuf, 0);
+	if (!ptr) {
+		pr_err("dma_buf_kmap failed\n");
+		err = -ENOMEM;
+		goto err;
+	}
+	if (memchr_inv(ptr, 0xc5, PAGE_SIZE)) {
+		dma_buf_kunmap(dmabuf, 0, ptr);
+		pr_err("Exported page[0] did not retain 0xc5!\n");
+		err = -EINVAL;
+		goto err;
+	}
+	dma_buf_kunmap(dmabuf, 0, ptr);
+
+	ptr = dma_buf_kmap(dmabuf, 2);
+	if (ptr) {
+		pr_err("Erroneously kmapped beyond the end of the object!\n");
+		dma_buf_kunmap(dmabuf, 2, ptr);
+		err = -EINVAL;
+		goto err;
+	}
+
+	ptr = dma_buf_kmap(dmabuf, -1);
+	if (ptr) {
+		pr_err("Erroneously kmapped before the start of the object!\n");
+		dma_buf_kunmap(dmabuf, -1, ptr);
+		err = -EINVAL;
+		goto err;
+	}
+
+	err = 0;
+err:
+	dma_buf_put(dmabuf);
+	return err;
+}
+
 int i915_gem_dmabuf_mock_selftests(void)
 {
 	static const struct i915_subtest tests[] = {
@@ -279,6 +378,7 @@ int i915_gem_dmabuf_mock_selftests(void)
 		SUBTEST(igt_dmabuf_import),
 		SUBTEST(igt_dmabuf_import_ownership),
 		SUBTEST(igt_dmabuf_export_vmap),
+		SUBTEST(igt_dmabuf_export_kmap),
 	};
 	struct drm_i915_private *i915;
 	int err;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index 14e9c2fbc4e6..5ea373221f49 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -304,7 +304,7 @@ static int igt_evict_vm(void *arg)
 		goto cleanup;
 
 	/* Everything is pinned, nothing should happen */
-	err = i915_gem_evict_vm(&ggtt->base, false);
+	err = i915_gem_evict_vm(&ggtt->base);
 	if (err) {
 		pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
 		       err);
@@ -313,7 +313,7 @@ static int igt_evict_vm(void *arg)
 
 	unpin_ggtt(i915);
 
-	err = i915_gem_evict_vm(&ggtt->base, false);
+	err = i915_gem_evict_vm(&ggtt->base);
 	if (err) {
 		pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
 		       err);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 67d82bf1407f..8f011c447e41 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -266,7 +266,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj,
 		if (offset >= obj->base.size)
 			continue;
 
-		i915_gem_object_flush_gtt_write_domain(obj);
+		flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 
 		p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
 		cpu = kmap(p) + offset_in_page(offset);
@@ -545,7 +545,9 @@ static int igt_mmap_offset_exhaustion(void *arg)
 		}
 
 		mutex_lock(&i915->drm.struct_mutex);
+		intel_runtime_pm_get(i915);
 		err = make_obj_busy(obj);
+		intel_runtime_pm_put(i915);
 		mutex_unlock(&i915->drm.struct_mutex);
 		if (err) {
 			pr_err("[loop %d] Failed to busy the object\n", loop);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_request.c b/drivers/gpu/drm/i915/selftests/i915_gem_request.c
index 98b7aac41eec..6664cb2eb0b8 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_request.c
@@ -580,7 +580,7 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
 	if (err)
 		goto err;
 
-	err = i915_gem_object_set_to_gtt_domain(obj, true);
+	err = i915_gem_object_set_to_wc_domain(obj, true);
 	if (err)
 		goto err;
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
new file mode 100644
index 000000000000..7a44dab631b8
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "../i915_selftest.h"
+#include "i915_random.h"
+
+#include "mock_gem_device.h"
+#include "mock_timeline.h"
+
+struct __igt_sync {
+	const char *name;
+	u32 seqno;
+	bool expected;
+	bool set;
+};
+
+static int __igt_sync(struct intel_timeline *tl,
+		      u64 ctx,
+		      const struct __igt_sync *p,
+		      const char *name)
+{
+	int ret;
+
+	if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {
+		pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n",
+		       name, p->name, ctx, p->seqno, yesno(p->expected));
+		return -EINVAL;
+	}
+
+	if (p->set) {
+		ret = __intel_timeline_sync_set(tl, ctx, p->seqno);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int igt_sync(void *arg)
+{
+	const struct __igt_sync pass[] = {
+		{ "unset", 0, false, false },
+		{ "new", 0, false, true },
+		{ "0a", 0, true, true },
+		{ "1a", 1, false, true },
+		{ "1b", 1, true, true },
+		{ "0b", 0, true, false },
+		{ "2a", 2, false, true },
+		{ "4", 4, false, true },
+		{ "INT_MAX", INT_MAX, false, true },
+		{ "INT_MAX-1", INT_MAX-1, true, false },
+		{ "INT_MAX+1", (u32)INT_MAX+1, false, true },
+		{ "INT_MAX", INT_MAX, true, false },
+		{ "UINT_MAX", UINT_MAX, false, true },
+		{ "wrap", 0, false, true },
+		{ "unwrap", UINT_MAX, true, false },
+		{},
+	}, *p;
+	struct intel_timeline *tl;
+	int order, offset;
+	int ret;
+
+	tl = mock_timeline(0);
+	if (!tl)
+		return -ENOMEM;
+
+	for (p = pass; p->name; p++) {
+		for (order = 1; order < 64; order++) {
+			for (offset = -1; offset <= (order > 1); offset++) {
+				u64 ctx = BIT_ULL(order) + offset;
+
+				ret = __igt_sync(tl, ctx, p, "1");
+				if (ret)
+					goto out;
+			}
+		}
+	}
+	mock_timeline_destroy(tl);
+
+	tl = mock_timeline(0);
+	if (!tl)
+		return -ENOMEM;
+
+	for (order = 1; order < 64; order++) {
+		for (offset = -1; offset <= (order > 1); offset++) {
+			u64 ctx = BIT_ULL(order) + offset;
+
+			for (p = pass; p->name; p++) {
+				ret = __igt_sync(tl, ctx, p, "2");
+				if (ret)
+					goto out;
+			}
+		}
+	}
+
+out:
+	mock_timeline_destroy(tl);
+	return ret;
+}
+
+static unsigned int random_engine(struct rnd_state *rnd)
+{
+	return ((u64)prandom_u32_state(rnd) * I915_NUM_ENGINES) >> 32;
+}
+
+static int bench_sync(void *arg)
+{
+	struct rnd_state prng;
+	struct intel_timeline *tl;
+	unsigned long end_time, count;
+	u64 prng32_1M;
+	ktime_t kt;
+	int order, last_order;
+
+	tl = mock_timeline(0);
+	if (!tl)
+		return -ENOMEM;
+
+	/* Lookups from cache are very fast and so the random number generation
+	 * and the loop itself becomes a significant factor in the per-iteration
+	 * timings. We try to compensate the results by measuring the overhead
+	 * of the prng and subtract it from the reported results.
+	 */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		u32 x;
+
+		/* Make sure the compiler doesn't optimise away the prng call */
+		WRITE_ONCE(x, prandom_u32_state(&prng));
+
+		count++;
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	pr_debug("%s: %lu random evaluations, %lluns/prng\n",
+		 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+	prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count);
+
+	/* Benchmark (only) setting random context ids */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		u64 id = i915_prandom_u64_state(&prng);
+
+		__intel_timeline_sync_set(tl, id, 0);
+		count++;
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
+	pr_info("%s: %lu random insertions, %lluns/insert\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	/* Benchmark looking up the exact same context ids as we just set */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	end_time = count;
+	kt = ktime_get();
+	while (end_time--) {
+		u64 id = i915_prandom_u64_state(&prng);
+
+		if (!__intel_timeline_sync_is_later(tl, id, 0)) {
+			mock_timeline_destroy(tl);
+			pr_err("Lookup of %llu failed\n", id);
+			return -EINVAL;
+		}
+	}
+	kt = ktime_sub(ktime_get(), kt);
+	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
+	pr_info("%s: %lu random lookups, %lluns/lookup\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	mock_timeline_destroy(tl);
+	cond_resched();
+
+	tl = mock_timeline(0);
+	if (!tl)
+		return -ENOMEM;
+
+	/* Benchmark setting the first N (in order) contexts */
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		__intel_timeline_sync_set(tl, count++, 0);
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	pr_info("%s: %lu in-order insertions, %lluns/insert\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	/* Benchmark looking up the exact same context ids as we just set */
+	end_time = count;
+	kt = ktime_get();
+	while (end_time--) {
+		if (!__intel_timeline_sync_is_later(tl, end_time, 0)) {
+			pr_err("Lookup of %lu failed\n", end_time);
+			mock_timeline_destroy(tl);
+			return -EINVAL;
+		}
+	}
+	kt = ktime_sub(ktime_get(), kt);
+	pr_info("%s: %lu in-order lookups, %lluns/lookup\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	mock_timeline_destroy(tl);
+	cond_resched();
+
+	tl = mock_timeline(0);
+	if (!tl)
+		return -ENOMEM;
+
+	/* Benchmark searching for a random context id and maybe changing it */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		u32 id = random_engine(&prng);
+		u32 seqno = prandom_u32_state(&prng);
+
+		if (!__intel_timeline_sync_is_later(tl, id, seqno))
+			__intel_timeline_sync_set(tl, id, seqno);
+
+		count++;
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
+	pr_info("%s: %lu repeated insert/lookups, %lluns/op\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+	mock_timeline_destroy(tl);
+	cond_resched();
+
+	/* Benchmark searching for a known context id and changing the seqno */
+	for (last_order = 1, order = 1; order < 32;
+	     ({ int tmp = last_order; last_order = order; order += tmp; })) {
+		unsigned int mask = BIT(order) - 1;
+
+		tl = mock_timeline(0);
+		if (!tl)
+			return -ENOMEM;
+
+		count = 0;
+		kt = ktime_get();
+		end_time = jiffies + HZ/10;
+		do {
+			/* Without assuming too many details of the underlying
+			 * implementation, try to identify its phase-changes
+			 * (if any)!
+			 */
+			u64 id = (u64)(count & mask) << order;
+
+			__intel_timeline_sync_is_later(tl, id, 0);
+			__intel_timeline_sync_set(tl, id, 0);
+
+			count++;
+		} while (!time_after(jiffies, end_time));
+		kt = ktime_sub(ktime_get(), kt);
+		pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n",
+			__func__, count, order,
+			(long long)div64_ul(ktime_to_ns(kt), count));
+		mock_timeline_destroy(tl);
+		cond_resched();
+	}
+
+	return 0;
+}
+
+int i915_gem_timeline_mock_selftests(void)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(igt_sync),
+		SUBTEST(bench_sync),
+	};
+
+	return i915_subtests(tests, NULL);
+}
diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
index be9a9ebf5692..fc74687501ba 100644
--- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
@@ -9,9 +9,12 @@
  * Tests are executed in order by igt/drv_selftest
  */
 selftest(sanitycheck, i915_mock_sanitycheck) /* keep first (igt selfcheck) */
+selftest(fence, i915_sw_fence_mock_selftests)
 selftest(scatterlist, scatterlist_mock_selftests)
+selftest(syncmap, i915_syncmap_mock_selftests)
 selftest(uncore, intel_uncore_mock_selftests)
 selftest(breadcrumbs, intel_breadcrumbs_mock_selftests)
+selftest(timelines, i915_gem_timeline_mock_selftests)
 selftest(requests, i915_gem_request_mock_selftests)
 selftest(objects, i915_gem_object_mock_selftests)
 selftest(dmabuf, i915_gem_dmabuf_mock_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_random.c b/drivers/gpu/drm/i915/selftests/i915_random.c
index c17c83c30637..d044bf9a6feb 100644
--- a/drivers/gpu/drm/i915/selftests/i915_random.c
+++ b/drivers/gpu/drm/i915/selftests/i915_random.c
@@ -30,6 +30,17 @@
 
 #include "i915_random.h"
 
+u64 i915_prandom_u64_state(struct rnd_state *rnd)
+{
+	u64 x;
+
+	x = prandom_u32_state(rnd);
+	x <<= 32;
+	x |= prandom_u32_state(rnd);
+
+	return x;
+}
+
 static inline u32 i915_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
 {
 	return upper_32_bits((u64)prandom_u32_state(state) * ep_ro);
diff --git a/drivers/gpu/drm/i915/selftests/i915_random.h b/drivers/gpu/drm/i915/selftests/i915_random.h
index b9c334ce6cd9..6c9379871384 100644
--- a/drivers/gpu/drm/i915/selftests/i915_random.h
+++ b/drivers/gpu/drm/i915/selftests/i915_random.h
@@ -41,6 +41,8 @@
 #define I915_RND_SUBSTATE(name__, parent__) \
 	struct rnd_state name__ = I915_RND_STATE_INITIALIZER(prandom_u32_state(&(parent__)))
 
+u64 i915_prandom_u64_state(struct rnd_state *rnd);
+
 unsigned int *i915_random_order(unsigned int count,
 				struct rnd_state *state);
 void i915_random_reorder(unsigned int *order,
diff --git a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
new file mode 100644
index 000000000000..19d145d6bf52
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
@@ -0,0 +1,582 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/completion.h>
+#include <linux/delay.h>
+
+#include "../i915_selftest.h"
+
+static int __i915_sw_fence_call
+fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+{
+	switch (state) {
+	case FENCE_COMPLETE:
+		break;
+
+	case FENCE_FREE:
+		/* Leave the fence for the caller to free it after testing */
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct i915_sw_fence *alloc_fence(void)
+{
+	struct i915_sw_fence *fence;
+
+	fence = kmalloc(sizeof(*fence), GFP_KERNEL);
+	if (!fence)
+		return NULL;
+
+	i915_sw_fence_init(fence, fence_notify);
+	return fence;
+}
+
+static void free_fence(struct i915_sw_fence *fence)
+{
+	i915_sw_fence_fini(fence);
+	kfree(fence);
+}
+
+static int __test_self(struct i915_sw_fence *fence)
+{
+	if (i915_sw_fence_done(fence))
+		return -EINVAL;
+
+	i915_sw_fence_commit(fence);
+	if (!i915_sw_fence_done(fence))
+		return -EINVAL;
+
+	i915_sw_fence_wait(fence);
+	if (!i915_sw_fence_done(fence))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int test_self(void *arg)
+{
+	struct i915_sw_fence *fence;
+	int ret;
+
+	/* Test i915_sw_fence signaling and completion testing */
+	fence = alloc_fence();
+	if (!fence)
+		return -ENOMEM;
+
+	ret = __test_self(fence);
+
+	free_fence(fence);
+	return ret;
+}
+
+static int test_dag(void *arg)
+{
+	struct i915_sw_fence *A, *B, *C;
+	int ret = -EINVAL;
+
+	/* Test detection of cycles within the i915_sw_fence graphs */
+	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
+		return 0;
+
+	A = alloc_fence();
+	if (!A)
+		return -ENOMEM;
+
+	if (i915_sw_fence_await_sw_fence_gfp(A, A, GFP_KERNEL) != -EINVAL) {
+		pr_err("recursive cycle not detected (AA)\n");
+		goto err_A;
+	}
+
+	B = alloc_fence();
+	if (!B) {
+		ret = -ENOMEM;
+		goto err_A;
+	}
+
+	i915_sw_fence_await_sw_fence_gfp(A, B, GFP_KERNEL);
+	if (i915_sw_fence_await_sw_fence_gfp(B, A, GFP_KERNEL) != -EINVAL) {
+		pr_err("single depth cycle not detected (BAB)\n");
+		goto err_B;
+	}
+
+	C = alloc_fence();
+	if (!C) {
+		ret = -ENOMEM;
+		goto err_B;
+	}
+
+	if (i915_sw_fence_await_sw_fence_gfp(B, C, GFP_KERNEL) == -EINVAL) {
+		pr_err("invalid cycle detected\n");
+		goto err_C;
+	}
+	if (i915_sw_fence_await_sw_fence_gfp(C, B, GFP_KERNEL) != -EINVAL) {
+		pr_err("single depth cycle not detected (CBC)\n");
+		goto err_C;
+	}
+	if (i915_sw_fence_await_sw_fence_gfp(C, A, GFP_KERNEL) != -EINVAL) {
+		pr_err("cycle not detected (BA, CB, AC)\n");
+		goto err_C;
+	}
+	if (i915_sw_fence_await_sw_fence_gfp(A, C, GFP_KERNEL) == -EINVAL) {
+		pr_err("invalid cycle detected\n");
+		goto err_C;
+	}
+
+	i915_sw_fence_commit(A);
+	i915_sw_fence_commit(B);
+	i915_sw_fence_commit(C);
+
+	ret = 0;
+	if (!i915_sw_fence_done(C)) {
+		pr_err("fence C not done\n");
+		ret = -EINVAL;
+	}
+	if (!i915_sw_fence_done(B)) {
+		pr_err("fence B not done\n");
+		ret = -EINVAL;
+	}
+	if (!i915_sw_fence_done(A)) {
+		pr_err("fence A not done\n");
+		ret = -EINVAL;
+	}
+err_C:
+	free_fence(C);
+err_B:
+	free_fence(B);
+err_A:
+	free_fence(A);
+	return ret;
+}
+
+static int test_AB(void *arg)
+{
+	struct i915_sw_fence *A, *B;
+	int ret;
+
+	/* Test i915_sw_fence (A) waiting on an event source (B) */
+	A = alloc_fence();
+	if (!A)
+		return -ENOMEM;
+	B = alloc_fence();
+	if (!B) {
+		ret = -ENOMEM;
+		goto err_A;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(A, B, GFP_KERNEL);
+	if (ret < 0)
+		goto err_B;
+	if (ret == 0) {
+		pr_err("Incorrectly reported fence A was complete before await\n");
+		ret = -EINVAL;
+		goto err_B;
+	}
+
+	ret = -EINVAL;
+	i915_sw_fence_commit(A);
+	if (i915_sw_fence_done(A))
+		goto err_B;
+
+	i915_sw_fence_commit(B);
+	if (!i915_sw_fence_done(B)) {
+		pr_err("Fence B is not done\n");
+		goto err_B;
+	}
+
+	if (!i915_sw_fence_done(A)) {
+		pr_err("Fence A is not done\n");
+		goto err_B;
+	}
+
+	ret = 0;
+err_B:
+	free_fence(B);
+err_A:
+	free_fence(A);
+	return ret;
+}
+
+static int test_ABC(void *arg)
+{
+	struct i915_sw_fence *A, *B, *C;
+	int ret;
+
+	/* Test a chain of fences, A waits on B who waits on C */
+	A = alloc_fence();
+	if (!A)
+		return -ENOMEM;
+
+	B = alloc_fence();
+	if (!B) {
+		ret = -ENOMEM;
+		goto err_A;
+	}
+
+	C = alloc_fence();
+	if (!C) {
+		ret = -ENOMEM;
+		goto err_B;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(A, B, GFP_KERNEL);
+	if (ret < 0)
+		goto err_C;
+	if (ret == 0) {
+		pr_err("Incorrectly reported fence B was complete before await\n");
+		goto err_C;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(B, C, GFP_KERNEL);
+	if (ret < 0)
+		goto err_C;
+	if (ret == 0) {
+		pr_err("Incorrectly reported fence C was complete before await\n");
+		goto err_C;
+	}
+
+	ret = -EINVAL;
+	i915_sw_fence_commit(A);
+	if (i915_sw_fence_done(A)) {
+		pr_err("Fence A completed early\n");
+		goto err_C;
+	}
+
+	i915_sw_fence_commit(B);
+	if (i915_sw_fence_done(B)) {
+		pr_err("Fence B completed early\n");
+		goto err_C;
+	}
+
+	if (i915_sw_fence_done(A)) {
+		pr_err("Fence A completed early (after signaling B)\n");
+		goto err_C;
+	}
+
+	i915_sw_fence_commit(C);
+
+	ret = 0;
+	if (!i915_sw_fence_done(C)) {
+		pr_err("Fence C not done\n");
+		ret = -EINVAL;
+	}
+	if (!i915_sw_fence_done(B)) {
+		pr_err("Fence B not done\n");
+		ret = -EINVAL;
+	}
+	if (!i915_sw_fence_done(A)) {
+		pr_err("Fence A not done\n");
+		ret = -EINVAL;
+	}
+err_C:
+	free_fence(C);
+err_B:
+	free_fence(B);
+err_A:
+	free_fence(A);
+	return ret;
+}
+
+static int test_AB_C(void *arg)
+{
+	struct i915_sw_fence *A, *B, *C;
+	int ret = -EINVAL;
+
+	/* Test multiple fences (AB) waiting on a single event (C) */
+	A = alloc_fence();
+	if (!A)
+		return -ENOMEM;
+
+	B = alloc_fence();
+	if (!B) {
+		ret = -ENOMEM;
+		goto err_A;
+	}
+
+	C = alloc_fence();
+	if (!C) {
+		ret = -ENOMEM;
+		goto err_B;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(A, C, GFP_KERNEL);
+	if (ret < 0)
+		goto err_C;
+	if (ret == 0) {
+		ret = -EINVAL;
+		goto err_C;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(B, C, GFP_KERNEL);
+	if (ret < 0)
+		goto err_C;
+	if (ret == 0) {
+		ret = -EINVAL;
+		goto err_C;
+	}
+
+	i915_sw_fence_commit(A);
+	i915_sw_fence_commit(B);
+
+	ret = 0;
+	if (i915_sw_fence_done(A)) {
+		pr_err("Fence A completed early\n");
+		ret = -EINVAL;
+	}
+
+	if (i915_sw_fence_done(B)) {
+		pr_err("Fence B completed early\n");
+		ret = -EINVAL;
+	}
+
+	i915_sw_fence_commit(C);
+	if (!i915_sw_fence_done(C)) {
+		pr_err("Fence C not done\n");
+		ret = -EINVAL;
+	}
+
+	if (!i915_sw_fence_done(B)) {
+		pr_err("Fence B not done\n");
+		ret = -EINVAL;
+	}
+
+	if (!i915_sw_fence_done(A)) {
+		pr_err("Fence A not done\n");
+		ret = -EINVAL;
+	}
+
+err_C:
+	free_fence(C);
+err_B:
+	free_fence(B);
+err_A:
+	free_fence(A);
+	return ret;
+}
+
+static int test_C_AB(void *arg)
+{
+	struct i915_sw_fence *A, *B, *C;
+	int ret;
+
+	/* Test multiple event sources (A,B) for a single fence (C) */
+	A = alloc_fence();
+	if (!A)
+		return -ENOMEM;
+
+	B = alloc_fence();
+	if (!B) {
+		ret = -ENOMEM;
+		goto err_A;
+	}
+
+	C = alloc_fence();
+	if (!C) {
+		ret = -ENOMEM;
+		goto err_B;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(C, A, GFP_KERNEL);
+	if (ret < 0)
+		goto err_C;
+	if (ret == 0) {
+		ret = -EINVAL;
+		goto err_C;
+	}
+
+	ret = i915_sw_fence_await_sw_fence_gfp(C, B, GFP_KERNEL);
+	if (ret < 0)
+		goto err_C;
+	if (ret == 0) {
+		ret = -EINVAL;
+		goto err_C;
+	}
+
+	ret = 0;
+	i915_sw_fence_commit(C);
+	if (i915_sw_fence_done(C))
+		ret = -EINVAL;
+
+	i915_sw_fence_commit(A);
+	i915_sw_fence_commit(B);
+
+	if (!i915_sw_fence_done(A)) {
+		pr_err("Fence A not done\n");
+		ret = -EINVAL;
+	}
+
+	if (!i915_sw_fence_done(B)) {
+		pr_err("Fence B not done\n");
+		ret = -EINVAL;
+	}
+
+	if (!i915_sw_fence_done(C)) {
+		pr_err("Fence C not done\n");
+		ret = -EINVAL;
+	}
+
+err_C:
+	free_fence(C);
+err_B:
+	free_fence(B);
+err_A:
+	free_fence(A);
+	return ret;
+}
+
+static int test_chain(void *arg)
+{
+	int nfences = 4096;
+	struct i915_sw_fence **fences;
+	int ret, i;
+
+	/* Test a long chain of fences */
+	fences = kmalloc_array(nfences, sizeof(*fences), GFP_KERNEL);
+	if (!fences)
+		return -ENOMEM;
+
+	for (i = 0; i < nfences; i++) {
+		fences[i] = alloc_fence();
+		if (!fences[i]) {
+			nfences = i;
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		if (i > 0) {
+			ret = i915_sw_fence_await_sw_fence_gfp(fences[i],
+							       fences[i - 1],
+							       GFP_KERNEL);
+			if (ret < 0) {
+				nfences = i + 1;
+				goto err;
+			}
+
+			i915_sw_fence_commit(fences[i]);
+		}
+	}
+
+	ret = 0;
+	for (i = nfences; --i; ) {
+		if (i915_sw_fence_done(fences[i])) {
+			if (ret == 0)
+				pr_err("Fence[%d] completed early\n", i);
+			ret = -EINVAL;
+		}
+	}
+	i915_sw_fence_commit(fences[0]);
+	for (i = 0; ret == 0 && i < nfences; i++) {
+		if (!i915_sw_fence_done(fences[i])) {
+			pr_err("Fence[%d] is not done\n", i);
+			ret = -EINVAL;
+		}
+	}
+
+err:
+	for (i = 0; i < nfences; i++)
+		free_fence(fences[i]);
+	kfree(fences);
+	return ret;
+}
+
+struct task_ipc {
+	struct work_struct work;
+	struct completion started;
+	struct i915_sw_fence *in, *out;
+	int value;
+};
+
+static void task_ipc(struct work_struct *work)
+{
+	struct task_ipc *ipc = container_of(work, typeof(*ipc), work);
+
+	complete(&ipc->started);
+
+	i915_sw_fence_wait(ipc->in);
+	smp_store_mb(ipc->value, 1);
+	i915_sw_fence_commit(ipc->out);
+}
+
+static int test_ipc(void *arg)
+{
+	struct task_ipc ipc;
+	int ret = 0;
+
+	/* Test use of i915_sw_fence as an interprocess signaling mechanism */
+	ipc.in = alloc_fence();
+	if (!ipc.in)
+		return -ENOMEM;
+	ipc.out = alloc_fence();
+	if (!ipc.out) {
+		ret = -ENOMEM;
+		goto err_in;
+	}
+
+	/* use a completion to avoid chicken-and-egg testing */
+	init_completion(&ipc.started);
+
+	ipc.value = 0;
+	INIT_WORK_ONSTACK(&ipc.work, task_ipc);
+	schedule_work(&ipc.work);
+
+	wait_for_completion(&ipc.started);
+
+	usleep_range(1000, 2000);
+	if (READ_ONCE(ipc.value)) {
+		pr_err("worker updated value before i915_sw_fence was signaled\n");
+		ret = -EINVAL;
+	}
+
+	i915_sw_fence_commit(ipc.in);
+	i915_sw_fence_wait(ipc.out);
+
+	if (!READ_ONCE(ipc.value)) {
+		pr_err("worker signaled i915_sw_fence before value was posted\n");
+		ret = -EINVAL;
+	}
+
+	flush_work(&ipc.work);
+	destroy_work_on_stack(&ipc.work);
+	free_fence(ipc.out);
+err_in:
+	free_fence(ipc.in);
+	return ret;
+}
+
+int i915_sw_fence_mock_selftests(void)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(test_self),
+		SUBTEST(test_dag),
+		SUBTEST(test_AB),
+		SUBTEST(test_ABC),
+		SUBTEST(test_AB_C),
+		SUBTEST(test_C_AB),
+		SUBTEST(test_chain),
+		SUBTEST(test_ipc),
+	};
+
+	return i915_subtests(tests, NULL);
+}
diff --git a/drivers/gpu/drm/i915/selftests/i915_syncmap.c b/drivers/gpu/drm/i915/selftests/i915_syncmap.c
new file mode 100644
index 000000000000..bcab3d00a785
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/i915_syncmap.c
@@ -0,0 +1,616 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "../i915_selftest.h"
+#include "i915_random.h"
+
+static char *
+__sync_print(struct i915_syncmap *p,
+	     char *buf, unsigned long *sz,
+	     unsigned int depth,
+	     unsigned int last,
+	     unsigned int idx)
+{
+	unsigned long len;
+	unsigned int i, X;
+
+	if (depth) {
+		unsigned int d;
+
+		for (d = 0; d < depth - 1; d++) {
+			if (last & BIT(depth - d - 1))
+				len = scnprintf(buf, *sz, "|   ");
+			else
+				len = scnprintf(buf, *sz, "    ");
+			buf += len;
+			*sz -= len;
+		}
+		len = scnprintf(buf, *sz, "%x-> ", idx);
+		buf += len;
+		*sz -= len;
+	}
+
+	/* We mark bits after the prefix as "X" */
+	len = scnprintf(buf, *sz, "0x%016llx", p->prefix << p->height << SHIFT);
+	buf += len;
+	*sz -= len;
+	X = (p->height + SHIFT) / 4;
+	scnprintf(buf - X, *sz + X, "%*s", X, "XXXXXXXXXXXXXXXXX");
+
+	if (!p->height) {
+		for_each_set_bit(i, (unsigned long *)&p->bitmap, KSYNCMAP) {
+			len = scnprintf(buf, *sz, " %x:%x,",
+					i, __sync_seqno(p)[i]);
+			buf += len;
+			*sz -= len;
+		}
+		buf -= 1;
+		*sz += 1;
+	}
+
+	len = scnprintf(buf, *sz, "\n");
+	buf += len;
+	*sz -= len;
+
+	if (p->height) {
+		for_each_set_bit(i, (unsigned long *)&p->bitmap, KSYNCMAP) {
+			buf = __sync_print(__sync_child(p)[i], buf, sz,
+					   depth + 1,
+					   last << 1 | !!(p->bitmap >> (i + 1)),
+					   i);
+		}
+	}
+
+	return buf;
+}
+
+static bool
+i915_syncmap_print_to_buf(struct i915_syncmap *p, char *buf, unsigned long sz)
+{
+	if (!p)
+		return false;
+
+	while (p->parent)
+		p = p->parent;
+
+	__sync_print(p, buf, &sz, 0, 1, 0);
+	return true;
+}
+
+static int check_syncmap_free(struct i915_syncmap **sync)
+{
+	i915_syncmap_free(sync);
+	if (*sync) {
+		pr_err("sync not cleared after free\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dump_syncmap(struct i915_syncmap *sync, int err)
+{
+	char *buf;
+
+	if (!err)
+		return check_syncmap_free(&sync);
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		goto skip;
+
+	if (i915_syncmap_print_to_buf(sync, buf, PAGE_SIZE))
+		pr_err("%s", buf);
+
+	kfree(buf);
+
+skip:
+	i915_syncmap_free(&sync);
+	return err;
+}
+
+static int igt_syncmap_init(void *arg)
+{
+	struct i915_syncmap *sync = (void *)~0ul;
+
+	/*
+	 * Cursory check that we can initialise a random pointer and transform
+	 * it into the root pointer of a syncmap.
+	 */
+
+	i915_syncmap_init(&sync);
+	return check_syncmap_free(&sync);
+}
+
+static int check_seqno(struct i915_syncmap *leaf, unsigned int idx, u32 seqno)
+{
+	if (leaf->height) {
+		pr_err("%s: not a leaf, height is %d\n",
+		       __func__, leaf->height);
+		return -EINVAL;
+	}
+
+	if (__sync_seqno(leaf)[idx] != seqno) {
+		pr_err("%s: seqno[%d], found %x, expected %x\n",
+		       __func__, idx, __sync_seqno(leaf)[idx], seqno);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int check_one(struct i915_syncmap **sync, u64 context, u32 seqno)
+{
+	int err;
+
+	err = i915_syncmap_set(sync, context, seqno);
+	if (err)
+		return err;
+
+	if ((*sync)->height) {
+		pr_err("Inserting first context=%llx did not return leaf (height=%d, prefix=%llx\n",
+		       context, (*sync)->height, (*sync)->prefix);
+		return -EINVAL;
+	}
+
+	if ((*sync)->parent) {
+		pr_err("Inserting first context=%llx created branches!\n",
+		       context);
+		return -EINVAL;
+	}
+
+	if (hweight32((*sync)->bitmap) != 1) {
+		pr_err("First bitmap does not contain a single entry, found %x (count=%d)!\n",
+		       (*sync)->bitmap, hweight32((*sync)->bitmap));
+		return -EINVAL;
+	}
+
+	err = check_seqno((*sync), ilog2((*sync)->bitmap), seqno);
+	if (err)
+		return err;
+
+	if (!i915_syncmap_is_later(sync, context, seqno)) {
+		pr_err("Lookup of first context=%llx/seqno=%x failed!\n",
+		       context, seqno);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int igt_syncmap_one(void *arg)
+{
+	I915_RND_STATE(prng);
+	IGT_TIMEOUT(end_time);
+	struct i915_syncmap *sync;
+	unsigned long max = 1;
+	int err;
+
+	/*
+	 * Check that inserting a new id, creates a leaf and only that leaf.
+	 */
+
+	i915_syncmap_init(&sync);
+
+	do {
+		u64 context = i915_prandom_u64_state(&prng);
+		unsigned long loop;
+
+		err = check_syncmap_free(&sync);
+		if (err)
+			goto out;
+
+		for (loop = 0; loop <= max; loop++) {
+			err = check_one(&sync, context,
+					prandom_u32_state(&prng));
+			if (err)
+				goto out;
+		}
+		max++;
+	} while (!__igt_timeout(end_time, NULL));
+	pr_debug("%s: Completed %lu single insertions\n",
+		 __func__, max * (max - 1) / 2);
+out:
+	return dump_syncmap(sync, err);
+}
+
+static int check_leaf(struct i915_syncmap **sync, u64 context, u32 seqno)
+{
+	int err;
+
+	err = i915_syncmap_set(sync, context, seqno);
+	if (err)
+		return err;
+
+	if ((*sync)->height) {
+		pr_err("Inserting context=%llx did not return leaf (height=%d, prefix=%llx\n",
+		       context, (*sync)->height, (*sync)->prefix);
+		return -EINVAL;
+	}
+
+	if (hweight32((*sync)->bitmap) != 1) {
+		pr_err("First entry into leaf (context=%llx) does not contain a single entry, found %x (count=%d)!\n",
+		       context, (*sync)->bitmap, hweight32((*sync)->bitmap));
+		return -EINVAL;
+	}
+
+	err = check_seqno((*sync), ilog2((*sync)->bitmap), seqno);
+	if (err)
+		return err;
+
+	if (!i915_syncmap_is_later(sync, context, seqno)) {
+		pr_err("Lookup of first entry context=%llx/seqno=%x failed!\n",
+		       context, seqno);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int igt_syncmap_join_above(void *arg)
+{
+	struct i915_syncmap *sync;
+	unsigned int pass, order;
+	int err;
+
+	i915_syncmap_init(&sync);
+
+	/*
+	 * When we have a new id that doesn't fit inside the existing tree,
+	 * we need to add a new layer above.
+	 *
+	 * 1: 0x00000001
+	 * 2: 0x00000010
+	 * 3: 0x00000100
+	 * 4: 0x00001000
+	 * ...
+	 * Each pass the common prefix shrinks and we have to insert a join.
+	 * Each join will only contain two branches, the latest of which
+	 * is always a leaf.
+	 *
+	 * If we then reuse the same set of contexts, we expect to build an
+	 * identical tree.
+	 */
+	for (pass = 0; pass < 3; pass++) {
+		for (order = 0; order < 64; order += SHIFT) {
+			u64 context = BIT_ULL(order);
+			struct i915_syncmap *join;
+
+			err = check_leaf(&sync, context, 0);
+			if (err)
+				goto out;
+
+			join = sync->parent;
+			if (!join) /* very first insert will have no parents */
+				continue;
+
+			if (!join->height) {
+				pr_err("Parent with no height!\n");
+				err = -EINVAL;
+				goto out;
+			}
+
+			if (hweight32(join->bitmap) != 2) {
+				pr_err("Join does not have 2 children: %x (%d)\n",
+				       join->bitmap, hweight32(join->bitmap));
+				err = -EINVAL;
+				goto out;
+			}
+
+			if (__sync_child(join)[__sync_branch_idx(join, context)] != sync) {
+				pr_err("Leaf misplaced in parent!\n");
+				err = -EINVAL;
+				goto out;
+			}
+		}
+	}
+out:
+	return dump_syncmap(sync, err);
+}
+
+static int igt_syncmap_join_below(void *arg)
+{
+	struct i915_syncmap *sync;
+	unsigned int step, order, idx;
+	int err;
+
+	i915_syncmap_init(&sync);
+
+	/*
+	 * Check that we can split a compacted branch by replacing it with
+	 * a join.
+	 */
+	for (step = 0; step < KSYNCMAP; step++) {
+		for (order = 64 - SHIFT; order > 0; order -= SHIFT) {
+			u64 context = step * BIT_ULL(order);
+
+			err = i915_syncmap_set(&sync, context, 0);
+			if (err)
+				goto out;
+
+			if (sync->height) {
+				pr_err("Inserting context=%llx (order=%d, step=%d) did not return leaf (height=%d, prefix=%llx\n",
+				       context, order, step, sync->height, sync->prefix);
+				err = -EINVAL;
+				goto out;
+			}
+		}
+	}
+
+	for (step = 0; step < KSYNCMAP; step++) {
+		for (order = SHIFT; order < 64; order += SHIFT) {
+			u64 context = step * BIT_ULL(order);
+
+			if (!i915_syncmap_is_later(&sync, context, 0)) {
+				pr_err("1: context %llx (order=%d, step=%d) not found\n",
+				       context, order, step);
+				err = -EINVAL;
+				goto out;
+			}
+
+			for (idx = 1; idx < KSYNCMAP; idx++) {
+				if (i915_syncmap_is_later(&sync, context + idx, 0)) {
+					pr_err("1: context %llx (order=%d, step=%d) should not exist\n",
+					       context + idx, order, step);
+					err = -EINVAL;
+					goto out;
+				}
+			}
+		}
+	}
+
+	for (order = SHIFT; order < 64; order += SHIFT) {
+		for (step = 0; step < KSYNCMAP; step++) {
+			u64 context = step * BIT_ULL(order);
+
+			if (!i915_syncmap_is_later(&sync, context, 0)) {
+				pr_err("2: context %llx (order=%d, step=%d) not found\n",
+				       context, order, step);
+				err = -EINVAL;
+				goto out;
+			}
+		}
+	}
+
+out:
+	return dump_syncmap(sync, err);
+}
+
+static int igt_syncmap_neighbours(void *arg)
+{
+	I915_RND_STATE(prng);
+	IGT_TIMEOUT(end_time);
+	struct i915_syncmap *sync;
+	int err;
+
+	/*
+	 * Each leaf holds KSYNCMAP seqno. Check that when we create KSYNCMAP
+	 * neighbouring ids, they all fit into the same leaf.
+	 */
+
+	i915_syncmap_init(&sync);
+	do {
+		u64 context = i915_prandom_u64_state(&prng) & ~MASK;
+		unsigned int idx;
+
+		if (i915_syncmap_is_later(&sync, context, 0)) /* Skip repeats */
+			continue;
+
+		for (idx = 0; idx < KSYNCMAP; idx++) {
+			err = i915_syncmap_set(&sync, context + idx, 0);
+			if (err)
+				goto out;
+
+			if (sync->height) {
+				pr_err("Inserting context=%llx did not return leaf (height=%d, prefix=%llx\n",
+				       context, sync->height, sync->prefix);
+				err = -EINVAL;
+				goto out;
+			}
+
+			if (sync->bitmap != BIT(idx + 1) - 1) {
+				pr_err("Inserting neighbouring context=0x%llx+%d, did not fit into the same leaf bitmap=%x (%d), expected %lx (%d)\n",
+				       context, idx,
+				       sync->bitmap, hweight32(sync->bitmap),
+				       BIT(idx + 1) - 1, idx + 1);
+				err = -EINVAL;
+				goto out;
+			}
+		}
+	} while (!__igt_timeout(end_time, NULL));
+out:
+	return dump_syncmap(sync, err);
+}
+
+static int igt_syncmap_compact(void *arg)
+{
+	struct i915_syncmap *sync;
+	unsigned int idx, order;
+	int err;
+
+	i915_syncmap_init(&sync);
+
+	/*
+	 * The syncmap are "space efficient" compressed radix trees - any
+	 * branch with only one child is skipped and replaced by the child.
+	 *
+	 * If we construct a tree with ids that are neighbouring at a non-zero
+	 * height, we form a join but each child of that join is directly a
+	 * leaf holding the single id.
+	 */
+	for (order = SHIFT; order < 64; order += SHIFT) {
+		err = check_syncmap_free(&sync);
+		if (err)
+			goto out;
+
+		/* Create neighbours in the parent */
+		for (idx = 0; idx < KSYNCMAP; idx++) {
+			u64 context = idx * BIT_ULL(order) + idx;
+
+			err = i915_syncmap_set(&sync, context, 0);
+			if (err)
+				goto out;
+
+			if (sync->height) {
+				pr_err("Inserting context=%llx (order=%d, idx=%d) did not return leaf (height=%d, prefix=%llx\n",
+				       context, order, idx,
+				       sync->height, sync->prefix);
+				err = -EINVAL;
+				goto out;
+			}
+		}
+
+		sync = sync->parent;
+		if (sync->parent) {
+			pr_err("Parent (join) of last leaf was not the sync!\n");
+			err = -EINVAL;
+			goto out;
+		}
+
+		if (sync->height != order) {
+			pr_err("Join does not have the expected height, found %d, expected %d\n",
+			       sync->height, order);
+			err = -EINVAL;
+			goto out;
+		}
+
+		if (sync->bitmap != BIT(KSYNCMAP) - 1) {
+			pr_err("Join is not full!, found %x (%d) expected %lx (%d)\n",
+			       sync->bitmap, hweight32(sync->bitmap),
+			       BIT(KSYNCMAP) - 1, KSYNCMAP);
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Each of our children should be a leaf */
+		for (idx = 0; idx < KSYNCMAP; idx++) {
+			struct i915_syncmap *leaf = __sync_child(sync)[idx];
+
+			if (leaf->height) {
+				pr_err("Child %d is a not leaf!\n", idx);
+				err = -EINVAL;
+				goto out;
+			}
+
+			if (leaf->parent != sync) {
+				pr_err("Child %d is not attached to us!\n",
+				       idx);
+				err = -EINVAL;
+				goto out;
+			}
+
+			if (!is_power_of_2(leaf->bitmap)) {
+				pr_err("Child %d holds more than one id, found %x (%d)\n",
+				       idx, leaf->bitmap, hweight32(leaf->bitmap));
+				err = -EINVAL;
+				goto out;
+			}
+
+			if (leaf->bitmap != BIT(idx)) {
+				pr_err("Child %d has wrong seqno idx, found %d, expected %d\n",
+				       idx, ilog2(leaf->bitmap), idx);
+				err = -EINVAL;
+				goto out;
+			}
+		}
+	}
+out:
+	return dump_syncmap(sync, err);
+}
+
+static int igt_syncmap_random(void *arg)
+{
+	I915_RND_STATE(prng);
+	IGT_TIMEOUT(end_time);
+	struct i915_syncmap *sync;
+	unsigned long count, phase, i;
+	u32 seqno;
+	int err;
+
+	i915_syncmap_init(&sync);
+
+	/*
+	 * Having tried to test the individual operations within i915_syncmap,
+	 * run a smoketest exploring the entire u64 space with random
+	 * insertions.
+	 */
+
+	count = 0;
+	phase = jiffies + HZ/100 + 1;
+	do {
+		u64 context = i915_prandom_u64_state(&prng);
+
+		err = i915_syncmap_set(&sync, context, 0);
+		if (err)
+			goto out;
+
+		count++;
+	} while (!time_after(jiffies, phase));
+	seqno = 0;
+
+	phase = 0;
+	do {
+		I915_RND_STATE(ctx);
+		u32 last_seqno = seqno;
+		bool expect;
+
+		seqno = prandom_u32_state(&prng);
+		expect = seqno_later(last_seqno, seqno);
+
+		for (i = 0; i < count; i++) {
+			u64 context = i915_prandom_u64_state(&ctx);
+
+			if (i915_syncmap_is_later(&sync, context, seqno) != expect) {
+				pr_err("context=%llu, last=%u this=%u did not match expectation (%d)\n",
+				       context, last_seqno, seqno, expect);
+				err = -EINVAL;
+				goto out;
+			}
+
+			err = i915_syncmap_set(&sync, context, seqno);
+			if (err)
+				goto out;
+		}
+
+		phase++;
+	} while (!__igt_timeout(end_time, NULL));
+	pr_debug("Completed %lu passes, each of %lu contexts\n", phase, count);
+out:
+	return dump_syncmap(sync, err);
+}
+
+int i915_syncmap_mock_selftests(void)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(igt_syncmap_init),
+		SUBTEST(igt_syncmap_one),
+		SUBTEST(igt_syncmap_join_above),
+		SUBTEST(igt_syncmap_join_below),
+		SUBTEST(igt_syncmap_neighbours),
+		SUBTEST(igt_syncmap_compact),
+		SUBTEST(igt_syncmap_random),
+	};
+
+	return i915_subtests(tests, NULL);
+}
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index ad56566e24db..fb9072d5877f 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -225,14 +225,6 @@ static bool assert_pin_valid(const struct i915_vma *vma,
 }
 
 __maybe_unused
-static bool assert_pin_e2big(const struct i915_vma *vma,
-			     const struct pin_mode *mode,
-			     int result)
-{
-	return result == -E2BIG;
-}
-
-__maybe_unused
 static bool assert_pin_enospc(const struct i915_vma *vma,
 			      const struct pin_mode *mode,
 			      int result)
@@ -255,7 +247,6 @@ static int igt_vma_pin1(void *arg)
 #define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " }
 #define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" }
 #define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL)
-#define TOOBIG(sz, fl) __INVALID(sz, fl, assert_pin_e2big, E2BIG)
 #define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC)
 		VALID(0, PIN_GLOBAL),
 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE),
@@ -276,11 +267,11 @@ static int igt_vma_pin1(void *arg)
 		VALID(8192, PIN_GLOBAL),
 		VALID(i915->ggtt.mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE),
 		VALID(i915->ggtt.mappable_end, PIN_GLOBAL | PIN_MAPPABLE),
-		TOOBIG(i915->ggtt.mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
+		NOSPACE(i915->ggtt.mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
 		VALID(i915->ggtt.base.total - 4096, PIN_GLOBAL),
 		VALID(i915->ggtt.base.total, PIN_GLOBAL),
-		TOOBIG(i915->ggtt.base.total + 4096, PIN_GLOBAL),
-		TOOBIG(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
+		NOSPACE(i915->ggtt.base.total + 4096, PIN_GLOBAL),
+		NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
 		INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (i915->ggtt.mappable_end - 4096)),
 		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (i915->ggtt.base.total - 4096)),
 		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)),
@@ -300,7 +291,6 @@ static int igt_vma_pin1(void *arg)
 #endif
 		{ },
 #undef NOSPACE
-#undef TOOBIG
 #undef INVALID
 #undef __INVALID
 #undef VALID
diff --git a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
index 19860a372d90..7276194c04f7 100644
--- a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
@@ -117,7 +117,7 @@ static int igt_random_insert_remove(void *arg)
 
 	mock_engine_reset(engine);
 
-	waiters = drm_malloc_gfp(count, sizeof(*waiters), GFP_TEMPORARY);
+	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
 	if (!waiters)
 		goto out_engines;
 
@@ -169,7 +169,7 @@ out_order:
 out_bitmap:
 	kfree(bitmap);
 out_waiters:
-	drm_free_large(waiters);
+	kvfree(waiters);
 out_engines:
 	mock_engine_flush(engine);
 	return err;
@@ -187,7 +187,7 @@ static int igt_insert_complete(void *arg)
 
 	mock_engine_reset(engine);
 
-	waiters = drm_malloc_gfp(count, sizeof(*waiters), GFP_TEMPORARY);
+	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
 	if (!waiters)
 		goto out_engines;
 
@@ -254,7 +254,7 @@ static int igt_insert_complete(void *arg)
 out_bitmap:
 	kfree(bitmap);
 out_waiters:
-	drm_free_large(waiters);
+	kvfree(waiters);
 out_engines:
 	mock_engine_flush(engine);
 	return err;
@@ -368,7 +368,7 @@ static int igt_wakeup(void *arg)
 
 	mock_engine_reset(engine);
 
-	waiters = drm_malloc_gfp(count, sizeof(*waiters), GFP_TEMPORARY);
+	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
 	if (!waiters)
 		goto out_engines;
 
@@ -454,7 +454,7 @@ out_waiters:
 		put_task_struct(waiters[n].tsk);
 	}
 
-	drm_free_large(waiters);
+	kvfree(waiters);
 out_engines:
 	mock_engine_flush(engine);
 	return err;
diff --git a/drivers/gpu/drm/i915/selftests/mock_context.c b/drivers/gpu/drm/i915/selftests/mock_context.c
index 8d3a90c3f8ac..f8b9cc212b02 100644
--- a/drivers/gpu/drm/i915/selftests/mock_context.c
+++ b/drivers/gpu/drm/i915/selftests/mock_context.c
@@ -40,10 +40,18 @@ mock_context(struct drm_i915_private *i915,
 	INIT_LIST_HEAD(&ctx->link);
 	ctx->i915 = i915;
 
+	ctx->vma_lut.ht_bits = VMA_HT_BITS;
+	ctx->vma_lut.ht_size = BIT(VMA_HT_BITS);
+	ctx->vma_lut.ht = kcalloc(ctx->vma_lut.ht_size,
+				  sizeof(*ctx->vma_lut.ht),
+				  GFP_KERNEL);
+	if (!ctx->vma_lut.ht)
+		goto err_free;
+
 	ret = ida_simple_get(&i915->context_hw_ida,
 			     0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
 	if (ret < 0)
-		goto err_free;
+		goto err_vma_ht;
 	ctx->hw_id = ret;
 
 	if (name) {
@@ -58,6 +66,8 @@ mock_context(struct drm_i915_private *i915,
 
 	return ctx;
 
+err_vma_ht:
+	kvfree(ctx->vma_lut.ht);
 err_free:
 	kfree(ctx);
 	return NULL;
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 0ad624a1db90..5b18a2dc19a8 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -52,11 +52,12 @@ static void hw_delay_complete(unsigned long data)
 	spin_unlock(&engine->hw_lock);
 }
 
-static int mock_context_pin(struct intel_engine_cs *engine,
-			    struct i915_gem_context *ctx)
+static struct intel_ring *
+mock_context_pin(struct intel_engine_cs *engine,
+		 struct i915_gem_context *ctx)
 {
 	i915_gem_context_get(ctx);
-	return 0;
+	return engine->buffer;
 }
 
 static void mock_context_unpin(struct intel_engine_cs *engine,
@@ -72,7 +73,6 @@ static int mock_request_alloc(struct drm_i915_gem_request *request)
 	INIT_LIST_HEAD(&mock->link);
 	mock->delay = 0;
 
-	request->ring = request->engine->buffer;
 	return 0;
 }
 
@@ -112,7 +112,6 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 	if (!ring)
 		return NULL;
 
-	ring->engine = engine;
 	ring->size = sz;
 	ring->effective_size = sz;
 	ring->vaddr = (void *)(ring + 1);
@@ -141,7 +140,7 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 
 	/* minimal engine setup for requests */
 	engine->base.i915 = i915;
-	engine->base.name = name;
+	snprintf(engine->base.name, sizeof(engine->base.name), "%s", name);
 	engine->base.id = id++;
 	engine->base.status_page.page_addr = (void *)(engine + 1);
 
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 9f24c5da3f8d..627e2aa09766 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -30,6 +30,7 @@
 #include "mock_gem_device.h"
 #include "mock_gem_object.h"
 #include "mock_gtt.h"
+#include "mock_uncore.h"
 
 void mock_device_flush(struct drm_i915_private *i915)
 {
@@ -73,6 +74,7 @@ static void mock_device_release(struct drm_device *dev)
 
 	destroy_workqueue(i915->wq);
 
+	kmem_cache_destroy(i915->priorities);
 	kmem_cache_destroy(i915->dependencies);
 	kmem_cache_destroy(i915->requests);
 	kmem_cache_destroy(i915->vmas);
@@ -119,6 +121,7 @@ struct drm_i915_private *mock_gem_device(void)
 		goto err;
 
 	device_initialize(&pdev->dev);
+	pdev->class = PCI_BASE_CLASS_DISPLAY << 16;
 	pdev->dev.release = release_dev;
 	dev_set_name(&pdev->dev, "mock");
 	dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
@@ -143,6 +146,7 @@ struct drm_i915_private *mock_gem_device(void)
 	mkwrite_device_info(i915)->gen = -1;
 
 	spin_lock_init(&i915->mm.object_stat_lock);
+	mock_uncore_init(i915);
 
 	init_waitqueue_head(&i915->gpu_error.wait_queue);
 	init_waitqueue_head(&i915->gpu_error.reset_queue);
@@ -184,12 +188,16 @@ struct drm_i915_private *mock_gem_device(void)
 	if (!i915->dependencies)
 		goto err_requests;
 
+	i915->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
+	if (!i915->priorities)
+		goto err_dependencies;
+
 	mutex_lock(&i915->drm.struct_mutex);
 	INIT_LIST_HEAD(&i915->gt.timelines);
 	err = i915_gem_timeline_init__global(i915);
 	if (err) {
 		mutex_unlock(&i915->drm.struct_mutex);
-		goto err_dependencies;
+		goto err_priorities;
 	}
 
 	mock_init_ggtt(i915);
@@ -209,6 +217,8 @@ struct drm_i915_private *mock_gem_device(void)
 err_engine:
 	for_each_engine(engine, i915, id)
 		mock_engine_free(engine);
+err_priorities:
+	kmem_cache_destroy(i915->priorities);
 err_dependencies:
 	kmem_cache_destroy(i915->dependencies);
 err_requests:
diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.c b/drivers/gpu/drm/i915/selftests/mock_timeline.c
new file mode 100644
index 000000000000..47b1f47c5812
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/mock_timeline.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "mock_timeline.h"
+
+struct intel_timeline *mock_timeline(u64 context)
+{
+	static struct lock_class_key class;
+	struct intel_timeline *tl;
+
+	tl = kzalloc(sizeof(*tl), GFP_KERNEL);
+	if (!tl)
+		return NULL;
+
+	__intel_timeline_init(tl, NULL, context, &class, "mock");
+
+	return tl;
+}
+
+void mock_timeline_destroy(struct intel_timeline *tl)
+{
+	__intel_timeline_fini(tl);
+	kfree(tl);
+}
diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.h b/drivers/gpu/drm/i915/selftests/mock_timeline.h
new file mode 100644
index 000000000000..c27ff4639b8b
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/mock_timeline.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __MOCK_TIMELINE__
+#define __MOCK_TIMELINE__
+
+#include "../i915_gem_timeline.h"
+
+struct intel_timeline *mock_timeline(u64 context);
+void mock_timeline_destroy(struct intel_timeline *tl);
+
+#endif /* !__MOCK_TIMELINE__ */
diff --git a/drivers/gpu/drm/i915/selftests/mock_uncore.c b/drivers/gpu/drm/i915/selftests/mock_uncore.c
new file mode 100644
index 000000000000..8ef14c7e5e38
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/mock_uncore.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "mock_uncore.h"
+
+#define __nop_write(x) \
+static void \
+nop_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { }
+__nop_write(8)
+__nop_write(16)
+__nop_write(32)
+
+#define __nop_read(x) \
+static u##x \
+nop_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { return 0; }
+__nop_read(8)
+__nop_read(16)
+__nop_read(32)
+__nop_read(64)
+
+void mock_uncore_init(struct drm_i915_private *i915)
+{
+	ASSIGN_WRITE_MMIO_VFUNCS(i915, nop);
+	ASSIGN_READ_MMIO_VFUNCS(i915, nop);
+}
diff --git a/drivers/gpu/drm/i915/selftests/mock_uncore.h b/drivers/gpu/drm/i915/selftests/mock_uncore.h
new file mode 100644
index 000000000000..d79aa3ca4d51
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/mock_uncore.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __MOCK_UNCORE_H
+#define __MOCK_UNCORE_H
+
+void mock_uncore_init(struct drm_i915_private *i915);
+
+#endif /* !__MOCK_UNCORE_H */
author	Linus Torvalds <torvalds@linux-foundation.org>	2017-07-09 18:48:37 -0700
committer	Linus Torvalds <torvalds@linux-foundation.org>	2017-07-09 18:48:37 -0700
commit	af3c8d98508d37541d4bf57f13a984a7f73a328c (patch)
tree	e8dd974d6ebccd38b1e373be8a5e4a2f8bf3c6ce /drivers/gpu/drm/i915
parent	afs: Add metadata xattrs (diff)
parent	drm: Remove unused drm_file parameter to drm_syncobj_replace_fence() (diff)
download	linux-dev-af3c8d98508d37541d4bf57f13a984a7f73a328c.tar.xz linux-dev-af3c8d98508d37541d4bf57f13a984a7f73a328c.zip